fix archivebox add

2025-12-29 02:58:49 -08:00
1043 changed files with 36359 additions and 183393 deletions
@@ -1,63 +1,25 @@
 .DS_Store
 ._*
-/.heartbeat.json
-/package.json
-/package-lock.json
-__pycache__
-**/__pycache__
 *.pyc
-*.pyo
-*.py[cod]
-*$py.class
+__pycache__/
 .mypy_cache/
 .pytest_cache/
-.ruff_cache/
-.uv-cache/
-.cache/
-.cache*/
 .github/
 .pdm-build/
-.pdm-build/**
 .pdm-python
 .eggs/
-*.egg-info/
 .git/
-!.git/
-.git/*
 .vscode/
 !.git/HEAD
-!.git/packed-refs
-!.git/refs/
-!.git/refs/heads/
-!.git/refs/heads/**
+!.git/refs/heads/*

 venv/
 .venv/
 .venv-old/
-.docker_venv/
 .docker-venv/
 node_modules/
-abx-dl/
-abxpkg/
-abx-plugins/
-abxbus/
 chrome/
 chromeprofile/
-chrome_profile/
-lib/
-out/
-tests/out/
-users/
-archive/
-crawls/
-snapshots/
-logs/
-archivebox-docker-smoke*/
-archivebox-compose-smoke*/
-docker-test/
-docker-test*/
-core
-*.core

 pdm.dev.lock
 pdm.lock
@@ -68,28 +30,13 @@ dist/
 brew_dist/
 deb_dist/
 pip_dist/
-cache/
-caches/
 assets/
 docker/
 website/
 typings/

-tmp/
-.tmp/
 data/
 data*/
-
-personas/
-sources/
 output/
 index.sqlite3
 index.sqlite3-wal
-queue.sqlite3
-*.sqlite*
-data.*
-.archivebox_id
-ArchiveBox.conf
-*.stdout
-*.stderr
-*.log
@@ -1,2 +1,2 @@
 github: ["ArchiveBox", "pirate"]
-custom: ["https://donate.archivebox.io", "https://swag.archivebox.io"]
+custom: ["https://donate.archivebox.io", "https://paypal.me/NicholasSweeting"]
@@ -27,10 +27,10 @@ body:
    id: steps_to_reproduce
    attributes:
      label: Steps to reproduce
-      description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
+      description: Please include the exact steps you took to trigger the issue.
      render: markdown
      placeholder: |
-        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
+        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:latest`
        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
        3. Typed 'https://example.com/xyz' into the 'Add URL' input field
        4. Clicked the 'Add+' button
@@ -69,13 +69,13 @@ body:
        0.8.6
        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
        
        Binary Dependencies:
-        √  python     3.14.0     venv_pip   ~/.venv/bin/python
-        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
+        √  python     3.11.9     venv_pip   ~/.venv/bin/python
+        √  django     5.1.2      venv_pip   ~/.venv/lib/python3.11/site-packages/django/__init__.py
+        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.11/site-packages/django/db/backends/sqlite3/base.py
        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
        ...
    validations:
@@ -93,7 +93,7 @@ body:
        - apt
        - brew
        - nix
-        - Docker (or Podman/LXC/K8s/TrueNAS/Proxmox/etc)
+        - Docker (or other container system like podman/LXC/Kubernetes or TrueNAS/Cloudron/YunoHost/etc.)
        - Other

  - type: dropdown
@@ -116,15 +116,15 @@ body:
    id: filesystem
    attributes:
      label: What type of drive are you using to store your ArchiveBox data?
-      description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` OR `data/archive`?
+      description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` or `data/archive` (e.g. NFS/SMB/CIFS/etc. or FUSE/RClone/S3/B2/OneDrive/etc.)?
      options:
-        - label: "some of `data/` is on a local SSD or NVMe drive"
+        - label: "`data/` is on a local SSD or NVMe drive"
          required: false
-        - label: "some of `data/` is on a spinning hard drive or external USB drive"
+        - label: "`data/` is on a spinning hard drive or external USB drive"
          required: false
-        - label: "some of `data/` is on a network mount (e.g. NFS/SMB/Ceph/GlusterFS/etc.)"
+        - label: "`data/` is on a network mount (e.g. NFS/SMB/CIFS/etc.)"
          required: false
-        - label: "some of `data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/Google Drive/Dropbox/etc.)"
+        - label: "`data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/OneDrive, etc.)"
          required: false
  

@@ -24,7 +24,7 @@ body:
    attributes:
      label: "What is the problem that your feature request solves?"
      description: |
-        Describe the problem or need that your feature request solves, feel free to include any screenshots or examples.
+        Describe the problem that your feature request solves, feel free to include any screenshots or examples.
      placeholder: |
        e.g. I need to be able to archive spanish and french subtitle files from a particular movie site https://example.com/somevideos that's going down soon.
    validations:
@@ -35,7 +35,7 @@ body:
    attributes:
      label: "What is your proposed solution?"
      description: |
-        Describe the ideal specific solution you'd want, *and whether it fits into any broader scope of changes*.
+        Describe the ideal specific solution you'd want, and whether it fits into any broader scope of changes.
      placeholder: |
        e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
        The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
@@ -47,7 +47,7 @@ body:
    attributes:
      label: "What hacks or alternative solutions have you tried to solve the problem?"
      description: |
-        A description of any alternative approaches, workarounds, or other solutions you've considered to fix the problem.
+        A clear and concise description of any alternative solutions, workarounds, or other software you've considered using to fix the problem.
      placeholder: |
        e.g. I wait for archivebox to finish archiving the page, then I manually run `yt-dlp --subs <URL>` inside 
        the `data/archive/<timestamp>/` directory to download the subtitle files and add them to the snapshot folder.
@@ -57,11 +57,10 @@ body:
  - type: textarea
    id: version
    attributes:
-      label: Share the entire output of the `archivebox version` command for the current version you are using.
+      label: What version of ArchiveBox are you currently using?
      description: |
-        DO NOT JUST ENTER "the latest version" OR YOUR ISSUE WILL BE CLOSED.  
-        We need to know what version of ArchiveBox and what feature flags you're currently running with in order to contextualize your feature request.  
-        Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in your specific environment.
+        We need to know what version of ArchiveBox and what OS you're currently using in order to contextualize your feature request.  
+        Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in specific environments/versions.
          
        Run the `archivebox version` command inside your current collection dir and paste the *full output* here (*not just the version number*).  
        For Docker Compose run: `docker compose run archivebox version`  
@@ -71,13 +70,13 @@ body:
        0.8.6
        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
        
        Binary Dependencies:
-        √  python     3.14.0     venv_pip   ~/.venv/bin/python
-        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
+        √  python     3.11.9     venv_pip   ~/.venv/bin/python
+        √  django     5.1.2      venv_pip   ~/.venv/lib/python3.11/site-packages/django/__init__.py
+        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.11/site-packages/django/db/backends/sqlite3/base.py
        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
        ...
    validations:
@@ -94,7 +93,7 @@ body:
          required: false
        - label: "It would be nice to have eventually"
          required: false
-        - label: "I'm willing to [start a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
+        - label: "I'm willing to [work on a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
          required: false
        - label: "I have [donated money](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations) to go towards fixing this issue"
          required: false
@@ -1,11 +1,8 @@
 blank_issues_enabled: false
 contact_links:
-  - name: ❓ Ask a question or start a discussion
-    url: https://github.com/ArchiveBox/ArchiveBox/discussions
-    about: "Ask a question, get support, or start a design discussion (to report a problem please use '🐞 Bug report' instead)"
-  - name: 💬 Chat with the dev team & community on Zulip
+  - name: 💬 Chat with us on Zulip
    url: https://zulip.archivebox.io
    about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
-  - name: 💁‍♂️ Hire us for professional support with fast response times
+  - name: 💁‍♂️ Hire us to provide archiving for your organization
    url: https://docs.monadical.com/s/archivebox-consulting-services
-    about: "We provide hosting, development, and support, including on-prem/cloud w/ SSO & storage, CAPTCHA-solving, proxies, etc."
+    about: "We provide hosting, develoment, and support, including on-prem/cloud setup w/ SSO & storage, CAPTCHA/rate-limiting avoidance, etc."
@@ -2,15 +2,18 @@

 ---

-## Security Documentation
+## Security Information

-Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
+Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
+
+https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+
+Also see this section of the README about important caveats when running ArchiveBox:
+
+https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
+
+You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:

- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
 - https://github.com/ArchiveBox/ArchiveBox#archive-layout
 - https://github.com/ArchiveBox/ArchiveBox#archivebox-development
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
@@ -29,44 +32,3 @@ We use Github's built-in [Private Reporting](https://docs.github.com/en/code-sec
 3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers

 You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).
-
---
-
-## CVEs and Report Credits
-
-> [!IMPORTANT]
-> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**  
-> Reporters **will still receive credit in release notes** however.
-
-> [!TIP]
-> Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.
-
-Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
-
---
-
-## Threat Model
-
-### All admin users are assumed to be trusted
-
-ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
-
- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
- non-logged in anonymous guest users
-
-Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
-
-### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
-
-Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
-
-Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
-
-### XSS, CSRF, CORS, CSP
-
-`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
-
-[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
-
-Note archivebox is designed to be able to archive any URL the server can reach, including internal intranet URLs and localhost. Users are expected to configure `URL_ALLOWLIST`/`URL_DENYLIST` accordingly if they need to limit what the server should archive.
-It does a basic check to try and avoid recursively archiving snapshots hosted by the archivebox server itself, but it's not a strict security check, it's just to prevent wasted CPU/disk from recursively archiving snapshots in a browser that has the ArchiveBox Browser Extension turned on.
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-set -Eeuo pipefail
-
-repo_name="$1"
-target_dir="${2:-$repo_name}"
-
-version="$(
-python3 - "$repo_name" <<'PY'
-import re
-import sys
-from pathlib import Path
-
-repo_name = sys.argv[1]
-lock_text = Path("uv.lock").read_text()
-match = re.search(
-    rf'^\[\[package\]\]\s*\nname = "{re.escape(repo_name)}"\s*\nversion = "([^"]+)"',
-    lock_text,
-    re.MULTILINE,
-)
-if not match:
-    raise SystemExit(f"Could not find {repo_name} in uv.lock")
-print(match.group(1))
-PY
-)"
-
-echo "Cloning ArchiveBox/${repo_name}@v${version} into ${target_dir}"
-git clone --depth=1 --branch "v${version}" "https://github.com/ArchiveBox/${repo_name}.git" "$target_dir"
@@ -1,49 +0,0 @@
-name: Claude Code
-
-on:
-  issue_comment:
-    types: [created]
-  pull_request_review_comment:
-    types: [created]
-  issues:
-    types: [opened, assigned]
-  pull_request_review:
-    types: [submitted]
-
-jobs:
-  claude:
-    if: |
-      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
-      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      pull-requests: write
-      issues: write
-      id-token: write
-      actions: read # Required for Claude to read CI results on PRs
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Run Claude Code
-        id: claude
-        uses: anthropics/claude-code-action@v1
-        with:
-          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
-
-          # This is an optional setting that allows Claude to read CI results on PRs
-          additional_permissions: |
-            actions: read
-
-          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
-          # prompt: 'Update the pull request description to include a summary of changes.'
-
-          # Optional: Add claude_args to customize behavior and configuration
-          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
-          # or https://code.claude.com/docs/en/cli-reference for available options
-          claude_args: '--allowed-tools Bash(gh pr:*)'
@@ -19,10 +19,6 @@ on:
  schedule:
    - cron: '33 17 * * 6'

-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
 jobs:
  analyze:
    name: Analyze (${{ matrix.language }})
@@ -0,0 +1,64 @@
+name: Build Debian package
+
+on:
+  workflow_dispatch:
+  push:
+
+env:
+  DEB_BUILD_OPTIONS: nocheck
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Install packaging dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y \
+            python3 python3-dev python3-pip python3-venv python3-all \
+            dh-python debhelper devscripts dput software-properties-common \
+            python3-distutils python3-setuptools python3-wheel python3-stdeb
+
+      # - name: Build Debian/Apt sdist_dsc
+      #   run: |
+      #     ./bin/build_pip.sh
+
+      # - name: Check ArchiveBox version
+      #   run: |
+      #     # must create dir needed for snaps to run as non-root on github actions
+      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
+      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
+      #     archivebox --version
+      #     archivebox init --setup
+
+      # - name: Add some links to test
+      #   run: |
+      #     cd "${{ github.workspace }}/data"
+      #     archivebox add 'https://example.com'
+      #     archivebox status
+
+      # - name: Commit built package
+      #   run: |
+      #     cd deb_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Debian package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/debian-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: deb_dist
+
+      # - name: Push build to Launchpad PPA
+      #   run: |
+      #     debsign -k "$PGP_KEY_ID" "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
+      #     dput archivebox "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
@@ -1,43 +0,0 @@
-name: Deploy Publicsite to GitHub Pages
-
-on:
-  push:
-    branches:
-      - dev
-    paths:
-      - publicsite/**
-      - .github/workflows/deploy-publicsite.yml
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-concurrency:
-  group: github-pages-publicsite
-  cancel-in-progress: true
-
-jobs:
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Setup Pages
-        uses: actions/configure-pages@v5
-
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
-        with:
-          path: ./publicsite
-
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4
@@ -5,42 +5,25 @@ on:
  push:
    branches:
      - '**'
+    tags:
+      - 'v*'
  # pull_request:

 env:
-  DOCKERHUB_IMAGE: archivebox/archivebox
-  GHCR_IMAGE: ghcr.io/archivebox/archivebox
-
-permissions:
-  contents: read
-  packages: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  DOCKER_IMAGE: archivebox-ci

 jobs:
-  build:
-    name: build ${{ matrix.platform }}
-    runs-on: ${{ matrix.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: linux/amd64
-            runner: ubuntu-24.04
-            cache_scope: docker-amd64
-            artifact_name: digest-linux-amd64
-          - platform: linux/arm64
-            runner: ubuntu-24.04-arm
-            cache_scope: docker-arm64
-            artifact_name: digest-linux-arm64
+  buildx:
+    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
-        # with:
-        #   submodules: true
-        #   fetch-depth: 1
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        id: buildx
@@ -48,6 +31,7 @@ jobs:
        with:
          version: latest
          install: true
+          platforms: linux/amd64,linux/arm64

      - name: Builder instance name
        run: echo ${{ steps.buildx.outputs.name }}
@@ -55,278 +39,69 @@ jobs:
      - name: Available platforms
        run: echo ${{ steps.buildx.outputs.platforms }}

-      - name: Verify published abx-dl image
-        id: abx_dl_image
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABX_DL_VERSION="$(python3 - <<'PY'
-          import re
-          import tomllib
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-

-          with open("pyproject.toml", "rb") as f:
-              deps = tomllib.load(f)["project"]["dependencies"]
-
-          for dep in deps:
-              match = re.match(r"abx-dl\s*(?:==|>=)\s*([^,;\s]+)", dep)
-              if match:
-                  print(match.group(1))
-                  break
-          else:
-              raise SystemExit("Missing abx-dl dependency in pyproject.toml")
-          PY
-          )"
-          ABX_DL_IMAGE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
-          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
-          docker buildx imagetools inspect "${ABX_DL_IMAGE}"
-
-      - name: Login to Docker Hub
+      - name: Docker Login
        uses: docker/login-action@v3
        if: github.event_name != 'pull_request'
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Collect Docker labels
+      - name: Collect Docker tags
+        # https://github.com/docker/metadata-action
        id: docker_meta
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$(python3 - <<'PY'
-          import tomllib
+        uses: docker/metadata-action@v5
+        with:
+          images: archivebox/archivebox,nikisweeting/archivebox
+          tags: |
+              # :stable
+              type=ref,event=branch
+              # :0.7.3
+              type=semver,pattern={{version}}
+              # :0.7
+              type=semver,pattern={{major}}.{{minor}}
+              # :sha-463ea54
+              type=sha
+              # :latest
+              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}

-          with open("pyproject.toml", "rb") as f:
-              print(tomllib.load(f)["project"]["version"])
-          PY
-          )"
-
-          {
-            echo 'labels<<EOF'
-            echo "org.opencontainers.image.version=${VERSION}"
-            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
-            echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
-            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
-            echo 'EOF'
-            echo "version=${VERSION}"
-          } >> "$GITHUB_OUTPUT"
-
-          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
-
-      - name: Build and push digest
+      - name: Build and push
        id: docker_build
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v5
        with:
          context: ./
          file: ./Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
          push: ${{ github.event_name != 'pull_request' }}
-          tags: |
-            ${{ env.DOCKERHUB_IMAGE }}
-            ${{ env.GHCR_IMAGE }}
+          tags: ${{ steps.docker_meta.outputs.tags }}
          labels: ${{ steps.docker_meta.outputs.labels }}
-          build-args: |
-            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
-          cache-from: type=gha,scope=${{ matrix.cache_scope }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
-          pull: true
-          platforms: ${{ matrix.platform }}
-          outputs: type=image,push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+          platforms: linux/amd64,linux/arm64

      - name: Image digest
        run: echo ${{ steps.docker_build.outputs.digest }}

-      - name: Validate pushed image version, commit, and size
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          IMAGE="${DOCKERHUB_IMAGE}@${{ steps.docker_build.outputs.digest }}"
-          SHORT_SHA="${GITHUB_SHA::7}"
-
-          DATA_DIR="$(mktemp -d)"
-          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
-          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version | tee /tmp/archivebox-version.out
-          docker run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
-          docker run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt > /tmp/archivebox-image-version.txt
-          grep -F "ArchiveBox v${{ steps.docker_meta.outputs.version }}" /tmp/archivebox-version.out
-          grep -F "COMMIT_HASH=${SHORT_SHA}" /tmp/archivebox-version.out
-          grep -Fx "COMMIT_HASH=${GITHUB_SHA}" /tmp/archivebox-image-version.txt
-
-          python3 - "$IMAGE" <<'PY'
-          import json
-          import subprocess
-          import sys
-
-          image = sys.argv[1]
-          limit = 780 * 1024 * 1024
-          manifest = json.loads(subprocess.check_output(["docker", "manifest", "inspect", image]))
-          total = manifest.get("config", {}).get("size", 0) + sum(layer.get("size", 0) for layer in manifest.get("layers", []))
-          print(f"{image} compressed_size={total / 1024 / 1024:.2f} MiB limit={limit / 1024 / 1024:.2f} MiB")
-          if total > limit:
-              raise SystemExit(f"{image} is over the compressed size limit")
-          PY
-
-      - name: Export digest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mkdir -p /tmp/digests
-          digest="${{ steps.docker_build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      - name: Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.artifact_name }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-  publish:
-    name: publish multiarch tags
-    runs-on: ubuntu-24.04
-    needs:
-      - build
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          version: latest
-          install: true
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: github.event_name != 'pull_request'
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Download digests
-        uses: actions/download-artifact@v4
-        with:
-          path: /tmp/digests
-          pattern: digest-*
-          merge-multiple: true
-
-      - name: Collect Docker tags
-        id: docker_meta
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$(python3 - <<'PY'
-          import tomllib
-
-          with open("pyproject.toml", "rb") as f:
-              print(tomllib.load(f)["project"]["version"])
-          PY
-          )"
-          BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
-          SHORT_SHA="${GITHUB_SHA::12}"
-          test -n "$BRANCH_TAG"
-          test -n "$SHORT_SHA"
-
-          {
-            echo 'dockerhub_tags<<EOF'
-            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
-              echo "${DOCKERHUB_IMAGE}:latest"
-            fi
-            echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
-            echo "${DOCKERHUB_IMAGE}:${VERSION}"
-            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
-            echo 'EOF'
-            echo 'ghcr_tags<<EOF'
-            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
-              echo "${GHCR_IMAGE}:latest"
-            fi
-            echo "${GHCR_IMAGE}:${BRANCH_TAG}"
-            echo "${GHCR_IMAGE}:${VERSION}"
-            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
-            echo 'EOF'
-            echo "version=${VERSION}"
-            echo "branch_tag=${BRANCH_TAG}"
-            echo "short_sha=${SHORT_SHA}"
-          } >> "$GITHUB_OUTPUT"
-
-          echo "[+] Publishing Docker Hub tags:"
-          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
-            printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
-          fi
-          printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
-          echo "[+] Publishing GHCR tags:"
-          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
-            printf '%s\n' "${GHCR_IMAGE}:latest"
-          fi
-          printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
-
-      - name: Create Docker Hub manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
-          [[ "${#DIGESTS[@]}" -gt 0 ]]
-
-          TAG_ARGS=()
-          while IFS= read -r tag; do
-            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
-          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
-
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do
-            REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}")
-          done
-
-          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Create GHCR manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
-          [[ "${#DIGESTS[@]}" -gt 0 ]]
-
-          TAG_ARGS=()
-          while IFS= read -r tag; do
-            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
-          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
-
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do
-            REFS+=("${GHCR_IMAGE}@sha256:${digest}")
-          done
-
-          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Inspect published images
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          while IFS= read -r tag; do
-            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
-          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
-          while IFS= read -r tag; do
-            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
-          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
-
      - name: Update README
        uses: peter-evans/dockerhub-description@v4
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          repository: archivebox/archivebox
+
+      # This ugly bit is necessary if you don't want your cache to grow forever
+      # until it hits GitHub's limit of 5GB.
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
@@ -1,59 +0,0 @@
-name: Duplicate Issue Detection
-
-on:
-  issues:
-    types: [opened]
-
-jobs:
-  check-duplicates:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Install opencode
-        run: curl -fsSL https://opencode.ai/install | bash
-
-      - name: Check for duplicate issues
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OPENCODE_PERMISSION: |
-            {
-              "bash": {
-                "gh issue*": "allow",
-                "gh pr*": "allow",
-                "*": "deny"
-              },
-              "webfetch": "allow"
-            }
-        run: |
-          opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created:
-
-          Issue number:
-          ${{ github.event.issue.number }}
-
-          Lookup this issue and search through existing issues and PRs (excluding #${{ github.event.issue.number }}) in this repository to find any potential duplicates of this new issue.
-          Consider:
-          1. Similar titles or descriptions
-          2. Same error messages or symptoms
-          3. Related functionality or components
-          4. Similar feature requests
-
-          If you find any potential duplicates, please comment on the new issue with:
-          - A brief explanation of why it might be a duplicate
-          - Links to the potentially duplicate issues or PRs
-          - A suggestion to check those issues first
-
-          Use this format for the comment:
-          'This issue might be a duplicate of existing issues. Please check:
-          - #[issue_number]: [brief description of similarity]
-
-          Feel free to ignore if none of these address your specific case.'
-
-          If no clear duplicates are found, do not comment."
@@ -0,0 +1,64 @@
+# Simple workflow for deploying static content to GitHub Pages
+name: Deploy static content to Pages
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["dev"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Single deploy job since we're just deploying
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Copy README.md into place
+        run: |
+          rm -f ./website/README.md
+          cp ./README.md ./website/README.md
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+
+      # - name: Checkout
+      #   uses: actions/checkout@v4
+      # - name: Setup Pages
+      #   uses: actions/configure-pages@v5
+      # - name: Upload artifact
+      #   uses: actions/upload-pages-artifact@v3
+      #   with:
+      #     # Upload entire repository
+      #     path: './website'
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
@@ -0,0 +1,51 @@
+name: Build Homebrew package
+
+on:
+  workflow_dispatch:
+  push:
+
+
+jobs:
+  build:
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      # TODO: modify archivebox.rb to update src url, hashes, and dependencies
+
+      - name: Build Homebrew Bottle
+        run: |
+          pip3 install --upgrade pip setuptools wheel
+          cd brew_dist/
+          brew install --build-bottle ./archivebox.rb
+          # brew bottle archivebox
+          archivebox version
+
+      - name: Add some links to test
+        run: |
+          mkdir data && cd data
+          archivebox init --setup
+          archivebox add 'https://example.com'
+          archivebox version
+          archivebox status
+
+      # - name: Commit built package
+      #   run: |
+      #     cd brew_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Homebrew package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/homebrew-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: brew_dist
+
+      # TODO: push bottle homebrew core PR with latest changes
@@ -0,0 +1,58 @@
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Build GitHub Pages website
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["dev"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Copy README.md into place
+        run: |
+          rm ./website/README.md
+          cp ./README.md ./website/README.md
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2
@@ -3,50 +3,33 @@ name: Run linters
 on:
  workflow_dispatch:
  push:
-    branches: ['**']
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true

 env:
-  UV_NO_SOURCES: "1"
-  PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
+  MAX_LINE_LENGTH: 110

 jobs:
  lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
        with:
          submodules: true
          fetch-depth: 1

-      - name: Clone abxpkg
-        run: bash .github/scripts/clone_abx_repo.sh abxpkg
-
-      - name: Clone abx-plugins
-        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
-
-      - name: Clone abx-dl
-        run: bash .github/scripts/clone_abx_repo.sh abx-dl
-
      - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v1
        with:
-          python-version: "3.13"
+          python-version: 3.9
          architecture: x64

-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          version: "0.10.6"
-          enable-cache: false
-
-      - name: Install dependencies with uv
+      - name: Install flake8
        run: |
-          uv venv
-          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[sonic,debug]"
+          pip install flake8

-      - name: Run prek
-        run: uv run --no-sync --no-sources prek run --all-files
+      - name: Lint with flake8
+        run: |
+          cd archivebox
+          # one pass for show-stopper syntax errors or undefined names
+          flake8 . --count --show-source --statistics
+          # one pass for small stylistic things
+          flake8 . --count --max-line-length="$MAX_LINE_LENGTH" --statistics
@@ -3,64 +3,76 @@ name: Build Pip package
 on:
  workflow_dispatch:
  push:
-    branches:
-      - '**'

-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  PYTHON_VERSION: "3.13"

 jobs:
  build:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-22.04
+
+    permissions:
+      id-token: write
+
    steps:
      - uses: actions/checkout@v4
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
        with:
-          version: "0.10.6"
-          enable-cache: false
+          submodules: true
+          fetch-depth: 1

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: '3.11'
          architecture: x64

-      - name: APT install archivebox dev + run dependencies
-        uses: awalsh128/cache-apt-pkgs-action@latest
+      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

-      - name: UV install archivebox dev + run sub-dependencies
-        env:
-          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
-        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+      - uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true

-      - name: UV build archivebox and archivebox/pkgs/* packages
-        run: |
-          uv build --all
+      - name: Install dependencies
+        run: pdm install --fail-fast --no-lock --dev --group=':all' --no-self

-      - name: UV install archivebox and archivebox/pkgs/* locally for tests
-        env:
-          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
+      - name: Build package
        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv pip install --python .venv/bin/python --no-deps dist/archivebox-*.whl
+          pdm build

-      - name: Verify built package full install
-        env:
-          UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
+      - name: Install from build
+        run: pip install ./dist/archivebox-*.whl
+
+      - name: Add some links to test
        run: |
-          set -Eeuo pipefail
-          DATA_DIR="$(mktemp -d)"
-          cd "$DATA_DIR"
-          uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox init
-          timeout 30m uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox install
-          uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox version
-          uv run --project "$GITHUB_WORKSPACE" --no-sync --no-sources archivebox status
+          mkdir data && cd data
+          archivebox init
+          archivebox add 'https://example.com'
+          archivebox version
+          archivebox status
+
+      #- name: Publish package distributions to PyPI
+      #  run: pdm publish --no-build
+
+      # - name: Push build to PyPI
+      #   run: |
+      #     cd pip_dist/
+      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
+      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
+
+      # - name: Commit built package
+      #   run: |
+      #     cd pip_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Pip package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/pip-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: pip_dist
+
@@ -1,52 +0,0 @@
-name: Release State
-
-on:
-  push:
-    branches:
-      - dev
-  workflow_dispatch:
-
-permissions:
-  contents: write
-  id-token: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-jobs:
-  release-state:
-    if: github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'release:')
-    runs-on: ubuntu-latest
-    environment: pypi
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          ref: ${{ github.ref_name }}
-          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@v6
-        with:
-          version: "0.10.6"
-          enable-cache: false
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Configure git identity
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-
-      - name: Run release script
-        env:
-          DEFAULT_BRANCH: dev
-          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-        run: ./bin/release.sh
@@ -1,350 +0,0 @@
-name: Parallel Tests
-
-on:
-  workflow_dispatch:
-  pull_request:
-    branches: [dev, main, master]
-  push:
-    branches: [dev]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-env:
-  PYTHONIOENCODING: utf-8
-  PYTHONLEGACYWINDOWSSTDIO: utf-8
-  USE_COLOR: False
-  UV_NO_SOURCES: "1"
-  ABXPKG_MIN_RELEASE_AGE: "0"
-
-jobs:
-  discover-tests:
-    name: Discover test files
-    runs-on: ubuntu-24.04
-    outputs:
-      test-files: ${{ steps.set-matrix.outputs.test-files }}
-      plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Discover test files
-        id: set-matrix
-        run: |
-          # Find all main test files
-          all_tests=$(find archivebox/tests -maxdepth 1 -name "test_*.py" -type f | sort)
-
-          # Create JSON array with test file info
-          json_array="["
-          first=true
-          for test_file in $all_tests; do
-            if [ "$first" = true ]; then
-              first=false
-            else
-              json_array+=","
-            fi
-
-            # Extract a display name for the test
-            name="main/$(basename $test_file .py | sed 's/^test_//')"
-            needs_chromium=false
-            if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then
-              needs_chromium=true
-            fi
-            needs_sonic=false
-            if grep -Eiq "shutil\\.which\\([\"']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic" "$test_file"; then
-              needs_sonic=true
-            fi
-
-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
-          done
-          json_array+="]"
-
-          echo "test-files=$json_array" >> $GITHUB_OUTPUT
-          echo "Found $(echo $all_tests | wc -w) test files"
-          echo "$json_array" | jq '.'
-
-      - name: Clone abx-plugins
-        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
-
-      - name: Discover plugin tests
-        id: set-plugin-matrix
-        run: |
-          plugin_tests=$(find abx-plugins/abx_plugins/plugins -maxdepth 2 -type d -name tests | sed 's#abx-plugins/abx_plugins/plugins/##; s#/tests##' | sort)
-
-          json_array="["
-          first=true
-          for plugin_name in $plugin_tests; do
-            if [ "$first" = true ]; then
-              first=false
-            else
-              json_array+=","
-            fi
-
-            needs_chromium=false
-            if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
-              needs_chromium=true
-            fi
-            needs_sonic=false
-            if [ "$plugin_name" = "search_backend_sonic" ]; then
-              needs_sonic=true
-            fi
-
-            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
-          done
-          json_array+="]"
-
-          echo "plugin-tests=$json_array" >> $GITHUB_OUTPUT
-          echo "Found $(echo $plugin_tests | wc -w) plugin test suites"
-          echo "$json_array" | jq '.'
-
-  run-tests:
-    name: ${{ matrix.test.name }}
-    runs-on: ubuntu-24.04
-    needs: discover-tests
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
-      CHROME_HEADLESS: "true"
-      PERSONAS_DIR: /tmp/abx-personas
-      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
-
-    strategy:
-      fail-fast: false
-      matrix:
-        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
-        python: ["3.13"]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Clone abxpkg
-        run: bash .github/scripts/clone_abx_repo.sh abxpkg
-
-      - name: Clone abx-plugins
-        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
-
-      - name: Clone abx-dl
-        run: bash .github/scripts/clone_abx_repo.sh abx-dl
-
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python }}
-          architecture: x64
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          version: "0.10.6"
-
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Cache uv
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-uv-
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.1
-
-      - name: Install dependencies with uv
-        run: |
-          uv venv
-          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
-
-      - name: Resolve core test binaries through abxpkg
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_INSTALL_TIMEOUT: "900"
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR"
-          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/wget/config.json:required_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/git/config.json:required_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_ripgrep/config.json:required_binaries" \
-            >> "$GITHUB_ENV"
-
-      - name: Resolve Chrome through abxpkg
-        if: ${{ matrix.test.needs_chromium }}
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_INSTALL_TIMEOUT: "900"
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR"
-          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/chrome/config.json:required_binaries" \
-            >> "$GITHUB_ENV"
-
-      - name: Resolve Sonic through abxpkg
-        if: ${{ matrix.test.needs_sonic }}
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_INSTALL_TIMEOUT: "900"
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR"
-          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_sonic/config.json:required_binaries" \
-            >> "$GITHUB_ENV"
-
-      - name: Run test - ${{ matrix.test.name }}
-        run: |
-          mkdir -p tests/out
-          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out
-
-  plugin-tests:
-    name: ${{ matrix.plugin.name }}
-    runs-on: ubuntu-24.04
-    needs: discover-tests
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
-      CHROME_HEADLESS: "true"
-      PERSONAS_DIR: /tmp/abx-personas
-      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
-
-    strategy:
-      fail-fast: false
-      matrix:
-        plugin: ${{ fromJson(needs.discover-tests.outputs.plugin-tests) }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Clone abxpkg
-        run: bash .github/scripts/clone_abx_repo.sh abxpkg
-
-      - name: Clone abx-plugins
-        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
-
-      - name: Clone abx-dl
-        run: bash .github/scripts/clone_abx_repo.sh abx-dl
-
-      - name: Set up Python 3.13
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-          architecture: x64
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          version: "0.10.6"
-
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Cache uv
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-3.13-uv-
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.1
-
-      - name: Install dependencies with uv
-        run: |
-          uv venv
-          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
-
-      - name: Resolve plugin dependencies through abxpkg
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_INSTALL_TIMEOUT: "900"
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR"
-          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/${{ matrix.plugin.plugin }}/config.json:required_binaries" \
-            >> "$GITHUB_ENV"
-
-      - name: Resolve Chrome through abxpkg
-        if: ${{ matrix.plugin.needs_chromium }}
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_INSTALL_TIMEOUT: "900"
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR"
-          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/chrome/config.json:required_binaries" \
-            >> "$GITHUB_ENV"
-
-      - name: Resolve Sonic through abxpkg
-        if: ${{ matrix.plugin.needs_sonic }}
-        env:
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_INSTALL_TIMEOUT: "900"
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR"
-          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-
-          uv run --no-sync --no-sources abxpkg env \
-            --install \
-            --lib="$LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/abx-plugins/abx_plugins/plugins/search_backend_sonic/config.json:required_binaries" \
-            >> "$GITHUB_ENV"
-
-      - name: Run plugin tests - ${{ matrix.plugin.name }}
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
-          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
-        run: |
-          uv run --no-sync --no-sources bash ./bin/test_plugins.sh "${{ matrix.plugin.plugin }}" --no-coverage
@@ -1,31 +1,21 @@
 name: Run tests
-
-on:
-  push:
-    branches: ['**']
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+on: [push]

 env:
  DOCKER_IMAGE: archivebox-ci
-  ARCHIVEBOX_IMAGE: archivebox-ci
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False
-  UV_NO_SOURCES: "1"

 jobs:
  python_tests:
    runs-on: ${{ matrix.os }}
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl

    strategy:
      matrix:
-        os: [ubuntu-24.04]
-        python: ["3.13"]
+        os: [ubuntu-22.04]
+        # os: [ubuntu-22.04, macos-latest, windows-latest]
+        python: [3.11]

    steps:
      - uses: actions/checkout@v4
@@ -33,15 +23,6 @@ jobs:
          submodules: true
          fetch-depth: 1

-      - name: Clone abxpkg
-        run: bash .github/scripts/clone_abx_repo.sh abxpkg
-
-      - name: Clone abx-plugins
-        run: bash .github/scripts/clone_abx_repo.sh abx-plugins
-
-      - name: Clone abx-dl
-        run: bash .github/scripts/clone_abx_repo.sh abx-dl
-
      ### Setup Python & JS Languages
      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@v4
@@ -49,34 +30,63 @@ jobs:
          python-version: ${{ matrix.python }}
          architecture: x64

-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          version: "0.10.6"
-
      - name: Set up Node JS
        uses: actions/setup-node@v4
        with:
-          node-version: 22
+          node-version: 20.10.0
+
+      - name: Setup PDM
+        uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true

      ### Install Python & JS Dependencies
-      - name: Cache uv
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: Cache pip
        uses: actions/cache@v3
+        id: cache-pip
        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-uv-
+            ${{ runner.os }}-${{ matrix.python }}-venv-

      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

-      - name: Install dependencies with uv
+      - name: Install pip dependencies
        run: |
-          uv venv
-          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
+          python -m pip install --upgrade pip setuptools wheel pytest bottle build
+          python -m pip install -r requirements.txt
+          python -m pip install -e .[sonic,ldap]
+
+      - name: Get npm cache dir
+        id: npm-cache
+        run: |
+          echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
+
+      - name: Cache npm
+        uses: actions/cache@v3
+        id: cache-npm
+        with:
+          path: ${{ steps.npm-cache.outputs.dir }}
+          key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node_modules
+
+      - name: Install npm requirements
+        run: |
+          npm install
+          echo "SINGLEFILE_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/single-file" >> $GITHUB_ENV
+          echo "READABILITY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor" >> $GITHUB_ENV
+          echo "MERCURY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/mercury-parser" >> $GITHUB_ENV

      ### Run the tests
      - name: Directory listing for debugging
@@ -84,33 +94,15 @@ jobs:
          pwd
          ls

-      - name: ArchiveBox full install check
+      - name: Archivebox version
        run: |
-          set -Eeuo pipefail
-          DATA_DIR="$PWD/tests/out/data"
-          mkdir -p "$DATA_DIR"
-          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox init
-          timeout 30m uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox install
-          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox version
-          uv run --directory "$DATA_DIR" --no-sync --no-sources archivebox status
+          archivebox version

      - name: Test built package with pytest
-        if: github.ref != 'refs/heads/dev'
+        # TODO: remove this exception for windows once we get tests passing on that platform
+        if: ${{ !contains(matrix.os, 'windows') }}
        run: |
-          mkdir -p tests/out
-          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out
-
-      - name: Run documentation code blocks
-        run: uv run --no-sync --no-sources pytest -vv --tb=long README.md AGENTS.md skills docs
-
-      - name: Run plugin tests
-        if: github.ref != 'refs/heads/dev'
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
-          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
-        run: |
-          uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage
+          python -m pytest -s --basetemp=tests/out --ignore=archivebox/pkgs

  docker_tests:
    runs-on: ubuntu-latest
@@ -121,6 +113,9 @@ jobs:
          submodules: true
          fetch-depth: 1

+      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
+      # - uses: satackey/action-docker-layer-caching@v0.0.8
+
      - name: Build image
        run: |
          docker build . -t "$DOCKER_IMAGE"
@@ -150,13 +145,14 @@ jobs:

      - name: Start docker-compose stack
        run: |
-          docker compose run --rm archivebox init
-          docker compose up -d --wait
+          docker-compose run archivebox init
+          docker-compose up -d
+          sleep 5
          curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
          curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'

      - name: Check added urls show up in index
        run: |
-          docker compose run --rm archivebox add --plugins=parse_txt_urls 'http://example.com/#test_docker'
+          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
          curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
-          docker compose down
+          docker-compose down || true
@@ -1,37 +0,0 @@
-name: Update downstream packages
-
-on:
-  push:
-    tags: ['v*']
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: update-downstream-packages-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  dispatch:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Dispatch downstream package updates
-        env:
-          DOWNSTREAM_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || secrets.HOMEBREW_TAP_TOKEN }}
-        run: |
-          if [ -z "$DOWNSTREAM_TOKEN" ]; then
-            echo "RELEASE_GH_TOKEN/HOMEBREW_TAP_TOKEN is not configured; downstream schedules will update packages."
-            exit 0
-          fi
-
-          for repo in homebrew-archivebox debian-archivebox; do
-            curl --fail-with-body \
-              -X POST \
-              -H "Accept: application/vnd.github+json" \
-              -H "Authorization: Bearer ${DOWNSTREAM_TOKEN}" \
-              -H "X-GitHub-Api-Version: 2022-11-28" \
-              "https://api.github.com/repos/ArchiveBox/${repo}/dispatches" \
-              -d '{"event_type":"archivebox-dev-updated","client_payload":{"ref":"dev"}}'
-          done
@@ -1,21 +1,11 @@
 .DS_Store
 ._*
-/.heartbeat.json
-/package.json
-/package-lock.json
 *.pyc
 __pycache__/
 .mypy_cache/
 .eggs/
 tests/out/

-# Coverage
-.coverage
-.coverage.*
-coverage.json
-coverage/
-htmlcov/
-
 # Python and Node dependencies
 venv/
 .venv/
@@ -28,7 +18,6 @@ pdm.dev.lock
 requirements-dev.txt

 # Packaging artifacts
-requirements.txt
 .pdm-python
 .pdm-build
 archivebox.egg-info
@@ -38,31 +27,16 @@ dist/

 # Data folders
 lib/
-out/
 tmp/
 data/
 data*/
-archivebox/tests/data/
-archive/
 output/
-logs/
-/-
-/personas/
-/sources/
 index.sqlite3
 queue.sqlite3
 *.sqlite*
 data.*
 .archivebox_id
-ArchiveBox.conf
-*.stdout
-*.stderr
-*.log
-.tmp/

 # vim
 *.sw?
 .vscode
-
-# Local Claude Code task locks (not project state)
-.claude/scheduled_tasks.lock
@@ -0,0 +1,3 @@
+[submodule "docs"]
+    path = docs
+    url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
@@ -1,67 +0,0 @@
-default_language_version:
-  python: python3.13
-
-repos:
-  - repo: https://github.com/asottile/yesqa
-    rev: v1.5.0
-    hooks:
-      - id: yesqa
-
-  - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.1
-    hooks:
-      - id: codespell
-        additional_dependencies:
-          - tomli
-
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.20.0
-    hooks:
-      - id: pyupgrade
-        args: [--py313-plus]
-
-  - repo: https://github.com/asottile/add-trailing-comma
-    rev: v3.1.0
-    hooks:
-      - id: add-trailing-comma
-
-  - repo: local
-    hooks:
-      - id: ruff-format
-        name: ruff-format
-        entry: uv run --active --no-sync --no-sources ruff format
-        language: system
-        types_or: [python, pyi]
-      - id: ruff-check
-        name: ruff-check
-        entry: uv run --active --no-sync --no-sources ruff check --fix
-        language: system
-        types_or: [python, pyi]
-
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
-    hooks:
-      - id: check-ast
-      - id: check-toml
-      - id: check-yaml
-        exclude: ^\.github/workflows/homebrew\.yml$
-      - id: check-json
-      - id: check-merge-conflict
-      - id: check-symlinks
-      - id: destroyed-symlinks
-      - id: check-case-conflict
-      - id: check-illegal-windows-names
-      - id: check-shebang-scripts-are-executable
-        exclude: ^(archivebox/.*\.py|archivebox/tests/.*\.py|archivebox/personas/export_browser_state\.js)$
-      - id: mixed-line-ending
-      - id: fix-byte-order-marker
-      - id: end-of-file-fixer
-      - id: detect-private-key
-      - id: debug-statements
-      - id: forbid-submodules
-        exclude: ^docs$
-      - id: check-added-large-files
-        args: ["--maxkb=600"]
-      - id: name-tests-test
-        args: ["--pytest-test-first"]
-        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)
@@ -1,85 +0,0 @@
-# ArchiveBox Agent Guide
-
-ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
-
-## Shared Standards
-
- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
- Start behavior fixes with a red failing test when a test is requested or practical.
- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
-
-## Development Setup
-
-```bash
-uv sync --dev --all-extras
-mkdir -p data
-cd data
-uv run --project .. archivebox init --install
-```
-
-Run collection commands from inside an initialized data directory:
-
-```bash
-cd data
-uv run --project .. archivebox status
-uv run --project .. archivebox add 'https://example.com'
-uv run --project .. archivebox run
-```
-
-## User-Facing Setup
-
-Recommended CLI install:
-
-```bash
-uv tool install --force .
-export PLUGINS=parse_txt_urls
-archivebox_data="$(mktemp -d)"
-cd "$archivebox_data"
-archivebox init --install
-archivebox add --plugins=parse_txt_urls 'https://example.com'
-```
-
-Alternative install methods:
-
- Docker Compose / Docker
- Homebrew
- Debian package
- pip
-
-## Basic Usage
-
-<!--pytest-codeblocks:cont-->
-```bash
-archivebox version
-archivebox help
-archivebox status
-archivebox install
-archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
-archivebox list --json --with-headers
-archivebox search 'example'
-archivebox update --filter-type=domain example.com
-archivebox remove --filter-type=exact 'https://example.com'
-archivebox run
-```
-
-## Verification
-
-Use targeted tests for focused work:
-
-```bash
-uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
-uv run prek run --all-files
-```
-
-Use the full release/deploy loop only when requested:
-
-```console
-./bin/release_dev_stack.sh
-```
@@ -1,188 +1,24 @@
-# syntax=docker/dockerfile:1.7
+# This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
+#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file
+# Usage:
+#     git submodule update --init --recursive
+#     git pull --recurse-submodules
+#     docker build . -t archivebox --no-cache
+#     docker run -v "$PWD/data":/data archivebox init
+#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
+#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
+#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
+# Multi-arch build:
+#     docker buildx create --use
+#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:0.7.5 -t archivebox/archivebox:dev
+#
+# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).

-# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns sonic, supervisor, Django, and the app runtime.
-# Build abx-dl first, then point this file at it:
-#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
-#       --build-context abxbus=../abxbus \
-#       --build-context abxpkg=../abxpkg \
-#       --build-context abx-plugins=../abx-plugins \
-#       -t archivebox/abx-dl:dev
-#   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
-#       -t archivebox:multistage
+#########################################################################################

-ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
-
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
-
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-ARG ARCHIVEBOX_COMMIT_HASH=""
-
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_NO_COMPILE=1 \
-    PIP_ONLY_BINARY=aiohttp \
-    npm_config_loglevel=error
-
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=24
-
-ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_ARCHIVEBOX_UID=911 \
-    DEFAULT_ARCHIVEBOX_GID=911 \
-    IN_DOCKER=True
-
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    CONFIG_DIR=/opt/archivebox \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
-    PERSONAS_DIR=/data/personas \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/venv/bin/python3 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-ENV HOME=/home/archivebox \
-    XDG_CONFIG_HOME=/home/archivebox/.config \
-    XDG_CACHE_HOME=/opt/archivebox/lib/cache \
-    ABXPKG_INSTALL_TIMEOUT=600 \
-    ABXPKG_POSTINSTALL_SCRIPTS=True \
-    ABXPKG_MIN_RELEASE_AGE=0 \
-    TIMEOUT=600
-
-ENV UV_COMPILE_BYTECODE=false \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv \
-    VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:$PATH"
-
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
-WORKDIR "$CODE_DIR"
-
-RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
-    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && abxpkg load --binproviders=env node \
-    && abxpkg load --binproviders=env uv \
-    ) | tee -a /VERSION.txt
-
-FROM archivebox-runtime-base AS archivebox-builder
-
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    <<'EOF'
-echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
-echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache
-echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends
-echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests
-rm -f /etc/apt/apt.conf.d/docker-clean
-apt-get update -qq
-apt-get install -qq -y --no-install-recommends \
-    build-essential gcc libldap2-dev libsasl2-dev libssl-dev
-/usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}"
-/usr/bin/uv pip install setuptools pip wheel
-
-mkdir -p /tmp/archivebox-uv-project
-/venv/bin/python - <<'PY'
-from pathlib import Path
-import json
-import re
-import urllib.request
-
-source = Path("/app/pyproject.toml")
-target = Path("/tmp/archivebox-uv-project/pyproject.toml")
-text = source.read_text()
-text = text.replace(
-    'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
-    'environments = ["sys_platform == \'linux\'"]',
-)
-
-# Docker builds need the just-published internal abx wheels immediately, but
-# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
-# Docker-only dependency view from the version JSON so the published package
-# metadata stays normal while image builds remain resumable after a release.
-for package in ("abxpkg", "abx-plugins", "abx-dl"):
-    match = re.search(rf'"{re.escape(package)}>=(?P<version>[^"]+)"', text)
-    if not match:
-        continue
-    version = match.group("version")
-    with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
-        data = json.load(response)
-    wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
-    text = re.sub(
-        rf'"{re.escape(package)}>=[^"]+"',
-        f'"{package} @ {wheel_url}"',
-        text,
-        count=1,
-    )
-
-target.write_text(text)
-PY
-
-/usr/bin/uv sync \
-    --project /tmp/archivebox-uv-project \
-    --refresh \
-    --no-dev \
-    --inexact \
-    --no-install-project \
-    --no-install-workspace \
-    --no-sources
-(find /venv/lib/python3.*/site-packages -type f -name '*.so' -exec strip --strip-unneeded {} + 2>/dev/null || true)
-rm -f /venv/bin/uv /venv/bin/uvx
-apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev
-apt-get autoremove -y
-rm -rf /var/lib/apt/lists/*
-EOF
-
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && COMMIT_HASH="$( \
-        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-            echo "$ARCHIVEBOX_COMMIT_HASH"; \
-        elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
-            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
-            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-                echo "$HEAD_REF"; \
-            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
-                REF_PATH="${HEAD_REF#ref: }"; \
-                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
-            fi; \
-        fi)" \
-    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
-    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
-    && rm -f /venv/bin/uv /venv/bin/uvx \
-    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
-
-FROM archivebox-runtime-base
+# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
+FROM python:3.11-slim-bookworm
+# FROM debian:bookworm-backports

 LABEL name="archivebox" \
    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -193,77 +29,305 @@ LABEL name="archivebox" \
    org.opencontainers.image.vendor="ArchiveBox" \
    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile"
+    com.docker.image.source.entrypoint="Dockerfile" \
+    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
+    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
+    com.docker.desktop.extension.api.version=">= 1.4.7" \
+    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
+    com.docker.extension.publisher-url="https://archivebox.io" \
+    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
+    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
+    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
+    com.docker.extension.categories='database,utility-tools'

-COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
+ARG TARGETPLATFORM
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT

-COPY --from=archivebox-builder /opt/uv/python /opt/uv/python
-COPY --from=archivebox-builder /venv /venv
-COPY --from=archivebox-builder /app /app
-COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
+######### Environment Variables #################################

-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
-    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
-    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
-    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
-    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
-    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
-    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
-    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
-    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
-    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
-    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
-    && abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
-    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
-    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
-    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$ABXPKG_LIB_DIR" \
-    && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+# Global system-level config
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    npm_config_loglevel=error

-WORKDIR "$DATA_DIR"
-RUN echo "[+] Initializing image collection..." \
-    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && HOME="$TMP_DIR" archivebox init \
-    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
-        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
-        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
-        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
-        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
-        "$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
-        2>/dev/null || true) \
-    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+# Version config
+ENV PYTHON_VERSION=3.11 \
+    NODE_VERSION=22

-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+# User config
+ENV ARCHIVEBOX_USER="archivebox" \
+    DEFAULT_PUID=911 \
+    DEFAULT_PGID=911 \
+    IN_DOCKER=True

-RUN --mount=type=cache,target=/tmp/abxpkg-cache,sharing=locked,mode=1777 \
-    chmod 1777 /tmp/abxpkg-cache \
-    && for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
-    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
-    && python3 -c 'from abx_dl.models import discover_plugins; [print(f"export {plugin.enabled_key}=True") for plugin in discover_plugins(runtime="archivebox").values() if plugin.enabled_key in plugin.config.properties]' > /tmp/archivebox-enable-plugins.env \
-    && sort /tmp/archivebox-enable-plugins.env | tee -a /VERSION.txt \
-    && source /tmp/archivebox-enable-plugins.env \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache ABXPKG_NO_CACHE=True ABXPKG_TMP_CACHE_DIR=/tmp/abxpkg-cache archivebox install \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups env HOME=/tmp/abxpkg-cache archivebox version 2>&1 | tee -a /VERSION.txt \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+# Global paths
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    PLAYWRIGHT_BROWSERS_PATH=/browsers
+    # GLOBAL_VENV=/venv \
+    # TODO: add TMP_DIR and LIB_DIR?

-RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
+# Build shell config
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
+
+######### System Environment ####################################
+
+# Detect ArchiveBox version number by reading pyproject.toml
+COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
+RUN grep '^version = ' "${CODE_DIR}/pyproject.toml" | awk -F'"' '{print $2}' > /VERSION.txt
+
+# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
+RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
+    && rm -f /etc/apt/apt.conf.d/docker-clean
+
+# Print debug info about build and save it to disk, for human eyes only, not used by anything else
+RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && echo \
+    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
+    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
+    && echo \
+    && uname -a \
+    && cat /etc/os-release | head -n7 \
+    && which bash && bash --version | head -n1 \
+    && which dpkg && dpkg --version | head -n1 \
+    && echo -e '\n\n' && env && echo -e '\n\n' \
    ) | tee -a /VERSION.txt

+# Create non-privileged user for archivebox and chrome
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
+    && groupadd --system $ARCHIVEBOX_USER \
+    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
+    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
+    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
+    | tee -a /VERSION.txt
+    # DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
+    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
+
+# Install system apt dependencies (adding backports to access more recent apt updates)
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
+    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' > /etc/apt/sources.list.d/backports.list \
+    && mkdir -p /etc/apt/keyrings \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports \
+        # 1. packaging dependencies
+        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
+        # 2. docker and init system dependencies
+        zlib1g-dev dumb-init gosu cron unzip grep dnsutils \
+        # 3. frivolous CLI helpers to make debugging failed archiving easier
+        tree nano iputils-ping \
+        # nano iputils-ping dnsutils htop procps jq yq
+    && rm -rf /var/lib/apt/lists/*
+
+# Install sonic search backend
+COPY --from=archivebox/sonic:1.4.9 /usr/local/bin/sonic /usr/local/bin/sonic
+COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
+RUN (which sonic && sonic --version) | tee -a /VERSION.txt
+
+######### Language Environments ####################################
+
+# Install Python environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
+    # NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
+    # && apt-get update -qq \
+    # && apt-get install -qq -y -t bookworm-backports --no-upgrade \
+    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
+    # && rm -rf /var/lib/apt/lists/* \
+    # tell PDM to allow using global system python site packages
+    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
+    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
+    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
+    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
+    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
+    # && python3 -m venv $GLOBAL_VENV \
+    # install global dependencies / python build dependencies in GLOBAL_VENV
+    # && pip install --upgrade pip setuptools wheel \
+    # Save version info
+    && ( \
+        which python3 && python3 --version | grep " $PYTHON_VERSION" \
+        && which pip && pip --version \
+        # && which pdm && pdm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+
+# Install Node environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Node $NODE_VERSION environment..." \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
+    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-upgrade libatomic1 \
+    && apt-get install -y -t bookworm-backports --no-upgrade \
+        nodejs \
+    && rm -rf /var/lib/apt/lists/* \
+    # Update NPM to latest version
+    && npm i -g npm --cache /root/.npm \
+    # Save version info
+    && ( \
+        which node && node --version \
+        && which npm && npm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+
+
+######### Extractor Dependencies ##################################
+
+# Install apt dependencies
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing APT extractor dependencies globally using apt..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports \
+        curl wget git ffmpeg ripgrep \
+        # Packages we have also needed in the past:
+        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
+    && rm -rf /var/lib/apt/lists/* \
+    # Save version info
+    && ( \
+        which curl && curl --version | head -n1 \
+        && which wget && wget --version 2>&1 | head -n1 \
+        && which git && git --version 2>&1 | head -n1 \
+        && which rg && rg --version 2>&1 | head -n1 \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+
+# Install chromium browser using playwright
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports \
+        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+        at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
+        libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
+        libxaw7 libxcomposite1 libxdamage1 libxfont2 \
+        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
+        # xfonts-scalable xfonts-utils xserver-common xvfb \
+        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
+        # libxss1 dbus dbus-x11 upower \
+    # && service dbus start \
+    # install Chromium using playwright
+    && pip install playwright \
+    && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
+    && playwright install chromium \
+    && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
+    && rm -rf /var/lib/apt/lists/* \
+    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
+    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \
+    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
+    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
+    # Save version info
+    && ( \
+        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install Node dependencies
+ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH"
+USER $ARCHIVEBOX_USER
+WORKDIR "/home/$ARCHIVEBOX_USER/.npm"
+RUN --mount=type=cache,target=/home/$ARCHIVEBOX_USER/.npm_cache,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT,uid=$DEFAULT_PUID,gid=$DEFAULT_PGID \
+    echo "[+] Installing NPM extractor dependencies in /home/$ARCHIVEBOX_USER/.npm..." \
+    && npm config set prefix "/home/$ARCHIVEBOX_USER/.npm" \
+    && npm install --global --prefer-offline --no-fund --no-audit --cache "/home/$ARCHIVEBOX_USER/.npm_cache" \
+        "@postlight/parser@^2.2.3" \
+        "readability-extractor@github:ArchiveBox/readability-extractor" \
+        "single-file-cli@^1.1.54" \
+        "puppeteer@^23.5.0" \
+        "@puppeteer/browsers@^2.4.0"
+USER root
+RUN ( \
+        which node && node --version \
+        && which npm && npm version \
+        && which postlight-parser \
+        && which readability-extractor && readability-extractor --version \
+        && which single-file && single-file --version \
+        && which puppeteer && puppeteer --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+######### Build Dependencies ####################################
+
+# Install ArchiveBox Python dependencies
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports \
+        build-essential gcc \
+        libssl-dev libldap2-dev libsasl2-dev \
+        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
+        pipx \
+    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
+    # && pdm use --venv in-project \
+    # && pdm run python -m ensurepip \
+    # && pdm sync --fail-fast --no-editable --group :all --no-self \
+    # && pdm export -o requirements.txt --without-hashes \
+    # && source $GLOBAL_VENV/bin/activate \
+    && pip install -r requirements.txt \
+    && apt-get purge -y \
+        build-essential gcc \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install ArchiveBox Python package from source
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
+    && pip install -e "${CODE_DIR}[all]" \
+    && rm -rf /var/lib/apt/lists/*
+
+####################################################
+
+# Setup ArchiveBox runtime config
+WORKDIR "$DATA_DIR"
+RUN openssl rand -hex 16 > /etc/machine-id \
+    && mkdir -p "/tmp/archivebox" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/tmp/archivebox" \
+    && mkdir -p "/usr/share/archivebox/lib" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/usr/share/archivebox/lib"
+
+ENV GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no \
+    TMP_DIR=/tmp/archivebox \
+    LIB_DIR=/usr/share/archivebox/lib \
+    ALLOWED_HOSTS=*
+
+# Print version for nice docker finish summary
+RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
+    ) | tee -a /VERSION.txt
+RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
+
+####################################################
+
+# Open up the interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
+    CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'

 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]
+CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]
@@ -0,0 +1,21 @@
+# Example: Using ArchiveBox in your own project
+
+FROM python:3.12-slim
+
+WORKDIR /app
+
+RUN pip install archivebox==0.8.5rc50
+RUN archivebox install
+
+RUN useradd -ms /bin/bash archivebox && chown -R archivebox /app
+
+
+
+
+
+
+
+
+
+
+
@@ -1,6 +1,6 @@
 MIT License

-Copyright (c) 2026 Nick Sweeting
+Copyright (c) 2024 Nick Sweeting

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -20,18 +20,16 @@ curl -fsSL 'https://get.archivebox.io' | bash    # (or see pip/brew/Docker instr
 <hr/>
 <br/>

-**ArchiveBox is a self-hosted app that lets you preserve content from websites in a variety of formats.**
+**ArchiveBox is a powerful, self-hosted internet archiving solution to collect, save, and view websites offline.**

-We aim to make your data immediately useful, and kept in formats that other programs can read directly. As output, we save standard HTML, PNG, PDF, TXT, JSON, WARC, SQLite, all guaranteed to be readable for decades to come. ArchiveBox also has a CLI, REST API, and webhooks so you can set up integrations with other services.
-
-Without active preservation effort, everything on the internet eventually disappears or degrades.
+Without active preservation effort, everything on the internet eventually disappears or degrades. Archive.org does a great job as a centralized service, but saved URLs have to be public, and they can't save every type of content.

 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>

-> ➡️ Get ArchiveBox with `uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.
+> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  

-*Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*
+*Once installed, it can be used as a [CLI tool](#usage), [self-hosted Web App](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python library](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [one-off command](#static-archive-exporting).*

 <br/>
 <hr/>
@@ -52,7 +50,7 @@ It also detects any content featured *inside* pages & extracts it out into a fol
 - 💾 **Github**/**Gitlab**/etc. links ➡️ `clone of GIT source code`, `README`, `images`, ...
 - ✨ *and more, see [Output Formats](#output-formats) below...*

-You can run ArchiveBox as a Docker web app to manage these snapshots, or continue accessing the same collection using the `uv`-installed CLI, Python API, and SQLite3 APIs.
+You can run ArchiveBox as a Docker web app to manage these snapshots, or continue accessing the same collection using the `pip`-installed CLI, Python API, and SQLite3 APIs. 
 All the ways of using it are equivalent, and provide matching features like adding tags, scheduling regular crawls, viewing logs, and more...

 <br/>
@@ -68,7 +66,7 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 <br/>


-**📦&nbsp; Install ArchiveBox using your preferred method: `docker` / `uv` / `brew` / `apt` / etc. ([see full Quickstart below](#quickstart)).**
+**📦&nbsp; Install ArchiveBox using your preferred method: `docker` / `pip` / `apt` / etc. ([see full Quickstart below](#quickstart)).**


 <details>
@@ -77,7 +75,7 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 <pre lang="bash"><code style="white-space: pre-line"># Option A: Get ArchiveBox with Docker Compose (recommended):
 mkdir -p ~/archivebox/data && cd ~/archivebox
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml   # edit options in this file as-needed
-docker compose run archivebox init
+docker compose run archivebox init --setup
 # docker compose run archivebox add 'https://example.com'
 # docker compose run archivebox help
 # docker compose up
@@ -85,17 +83,16 @@ docker compose run archivebox init
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -it -v $PWD:/data archivebox/archivebox:dev init
-# docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
-# docker run -it -v $PWD:/data archivebox/archivebox:dev help
-# docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
+docker run -it -v $PWD:/data archivebox/archivebox init --setup
+# docker run -it -v $PWD:/data archivebox/archivebox add 'https://example.com'
+# docker run -it -v $PWD:/data archivebox/archivebox help
+# docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox
 <br/>
 <br/>
-# Option C: Or install it with uv (see Quickstart below for apt, brew, and more)
-uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
+# Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
+pip install archivebox
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init
-archivebox install
+archivebox init --setup
 # archivebox add 'https://example.com'
 # archivebox help
 # archivebox server 0.0.0.0:8000
@@ -105,8 +102,7 @@ archivebox install
 curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
-<sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
-<sub>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
+<sub>Open <a href="http://localhost:8000"><code>http://localhost:8000</code></a> to see your server's Web UI ➡️</sub>
 </details>
 <br/>

@@ -134,7 +130,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
+- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
 - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
 - Advanced users: support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
@@ -154,9 +150,9 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 - **Individuals:**
  `saving bookmarks`, `preserving portfolio content`, `legacy / memoirs archival`
 - **Governments:**
-  `snapshotting public service sites`, `recordkeeping compliance`
+  `snapshoting public service sites`, `recordkeeping compliance`

-> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  
+> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  (we are also seeking [grant funding](https://github.com/ArchiveBox/ArchiveBox/issues/1126#issuecomment-1487431394))  
 > We offer: setup & support, CAPTCHA/ratelimit unblocking, SSO, audit logging/chain-of-custody, and more  
 > *ArchiveBox is a 🏛️ 501(c)(3) [nonprofit FSP](https://hackclub.com/hcb/) and all our work supports open-source development.* 

@@ -190,9 +186,9 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
 </code></pre></li>
 <li>Run the initial setup to create an admin user (or set ADMIN_USER/PASS in docker-compose.yml)
-<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init
+<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init --setup
 </code></pre></li>
-<li>Next steps: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
+<li>Next steps: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">docker compose up
 # completely optional, CLI can always be used without running a server
 # docker compose run [-T] archivebox [subcommand] [--help]
@@ -214,14 +210,14 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -v $PWD:/data -it archivebox/archivebox:dev init
+docker run -v $PWD:/data -it archivebox/archivebox init --setup
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
+<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
+<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox
 # completely optional, CLI can always be used without running a server
 # docker run -v $PWD:/data -it [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox:dev help
+docker run -v $PWD:/data -it archivebox/archivebox help
 </code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
 </li>
@@ -256,26 +252,27 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a


 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" height="28px" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="Pip" height="28px" align="top"/> <code>pip</code></b> (macOS/Linux/BSD)</summary>
 <br/>
 <ol>

-<li>Install <a href="https://docs.astral.sh/uv/getting-started/installation/">uv</a> on your system (if not already installed).</li>
-<li>Install the ArchiveBox package using <code>uv</code>.
-<pre lang="bash"><code style="white-space: pre-line">uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
+<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.10</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v18</a> on your system (if not already installed).</li>
+<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://pipx.pypa.io"><code>pipx</code></a>).
+<pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox yt-dlp playwright
+playwright install --with-deps chromium
 archivebox version
 # install any missing extras shown using apt/brew/pkg/etc. see Wiki for instructions
-#    python@3.13 node curl wget git ripgrep ...
+#    python@3.10 node curl wget git ripgrep ...
 </code></pre>
 <i>See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install">Install: Bare Metal</a> Wiki for full install instructions for each OS...</i>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data   # for example
-archivebox init     # instantialize a new collection
-archivebox install  # install all the runtime dependencies (e.g. chrome, single-file, yt-dlp, etc.)
+archivebox init --setup   # instantialize a new collection
+# (--setup auto-installs and link JS dependencies: singlefile, readability, mercury, etc.)
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
@@ -294,23 +291,29 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <details>
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
-<ol>
-<li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
-<pre lang="bash"><code style="white-space: pre-line">echo 'deb [trusted=yes] https://archivebox.github.io/debian-archivebox dev main' | sudo tee /etc/apt/sources.list.d/archivebox.list
+See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki for instructions. ➡️
+<!--<ol>
+<li>Add the ArchiveBox repository to your sources.<br/>
+<pre lang="bash"><code style="white-space: pre-line">echo "deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/archivebox.list
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C258F79DCC02E369
 sudo apt update
-sudo apt install archivebox
-archivebox version                         # make sure the package is installed
+</code></pre>
+</li>
+<li>Install the ArchiveBox package using <code>apt</code>.
+<pre lang="bash"><code style="white-space: pre-line">sudo apt install archivebox
+# update to newest version with pip (sometimes apt package is outdated)
+pip install --upgrade --ignore-installed archivebox yt-dlp playwright
+playwright install --with-deps chromium    # install chromium and its system dependencies
+archivebox version                         # make sure all dependencies are installed
 </code></pre>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init
-archivebox install
-archivebox add 'https://example.com'
+archivebox init --setup
 </code></pre>
 <br/>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
@@ -319,7 +322,7 @@ archivebox help
 </li>
 </ol>
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
-<sub>The apt package is a thin dev-channel wrapper around the normal Python install flow. See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for details, and run <code>sudo archivebox install</code> only if you want it to install missing system packages via apt.</sub>
+<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>-->
 <br/><br/>
 </details>

@@ -331,17 +334,19 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install the ArchiveBox package using <code>brew</code>.
 <pre lang="bash"><code style="white-space: pre-line">brew tap archivebox/archivebox
 brew install archivebox
+# update to newest version with pip (sometimes brew package is outdated)
+pip install --upgrade --ignore-installed archivebox yt-dlp playwright
+playwright install --with-deps chromium    # install chromium and its system dependencies
 archivebox version                         # make sure all dependencies are installed
 </code></pre>
 <i>See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki for more granular instructions for macOS... ➡️</i>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init
-archivebox install
+archivebox init --setup
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
@@ -362,7 +367,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 > *Warning: These are contributed by external volunteers and may lag behind the official `pip` channel.*

 <ul>
-<li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>, maintained by <a href="https://github.com/jasongodev"><code>@jasongodev</code></a>)</li>
+<li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>)</li>
 <li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
 <li>Nix: <a href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/archivebox/default.nix"><code>nix-env --install archivebox</code></a> (contributed by <a href="https://github.com/siraben"><code>@siraben</code></a>)</li>
 <li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
@@ -404,7 +409,6 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <ul>
 <li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
 <li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
-<li><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a></li>
 <li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
 <li><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a></li>
 <li><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a></li>
@@ -425,7 +429,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <ul>
 <li><a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">
 <img src="https://img.shields.io/badge/Custom_Development-ArchiveBox.io-%231a1a1a.svg?style=flat" height="22px"/>
-</a> (<a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">get hosting, support, and feature customization directly from us</a>)</li>
+</a> (<a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">get hosting, support, and feature customization directy from us</a>)</li>
 <li><a href="https://monadical.com">
 <img src="https://img.shields.io/badge/General_Dev_Consulting-Monadical.com-%231a1a1a.svg?style=flat" height="22px"/>
 </a> (<a href="https://monadical.com/contact-us.html">generalist consultancy that has ArchiveBox experience</a>)</li>
@@ -462,7 +466,6 @@ For more discussion on managed and paid hosting options see here: <a href="https
 #### ➡️&nbsp; Next Steps

 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
- (Optional) Create a persona and import browser cookies to archive logged-in sites: `archivebox persona create --import=chrome personal`
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -476,23 +479,7 @@ For more discussion on managed and paid hosting options see here: <a href="https
 ArchiveBox commands can be run in a terminal [directly on your host](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), or via [Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1)/[Docker Compose](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage).  
 <sup>(depending on how you chose to install it above)</sup>

-<!--
 ```bash
-set -euo pipefail
-__archivebox_docs_home="$(mktemp -d)"
-export HOME="$__archivebox_docs_home"
-export PLUGINS=parse_txt_urls
-mkdir -p ~/archivebox/data
-cd ~/archivebox/data
-archivebox init
-archivebox version
-archivebox help
-test -f index.sqlite3
-test -d archive
-rm -rf "$__archivebox_docs_home"
-```
-->
-```console
 mkdir -p ~/archivebox/data   # create a new data dir anywhere
 cd ~/archivebox/data         # IMPORTANT: cd into the directory

@@ -503,21 +490,15 @@ archivebox help
 # equivalent: docker compose run archivebox [subcommand] [--help]
 docker compose run archivebox help

-# equivalent: docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
-docker run -it -v $PWD:/data archivebox/archivebox:dev help
-
-# optional: import your browser cookies into a persona for logged-in archiving
-archivebox persona create --import=chrome personal
-# supported: chrome/chromium/brave/edge (Chromium-based only)
-# use --profile to target a specific profile (e.g. Default, Profile 1)
-# re-running import merges/dedupes cookies.txt (by domain/path/name) but replaces chrome_user_data
+# equivalent: docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
+docker run -it -v $PWD:/data archivebox/archivebox help
 ```

 #### ArchiveBox Subcommands

 - `archivebox` `help`/`version` to see the list of available subcommands / currently installed version info
 - `archivebox` `setup`/`init`/`config`/`status`/`shell`/`manage` to administer your collection
- `archivebox` `add`/`schedule` to pull in fresh URLs from [bookmarks/history/RSS/etc.](#input-formats)
+- `archivebox` `add`/`oneshot`/`schedule` to pull in fresh URLs from [bookmarks/history/RSS/etc.](#input-formats)
 - `archivebox` `list`/`update`/`remove` to manage existing Snapshots in your collection

 <br/>
@@ -525,15 +506,13 @@ archivebox persona create --import=chrome personal
 <summary><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" height="22px" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
-# make sure you have uv-installed ArchiveBox and it's available in your $PATH first
+# make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
 <br/>
 # archivebox [subcommand] [--help]
-mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init
-archivebox install
-archivebox add 'https://example.com'
+archivebox init --setup      # safe to run init multiple times (also how you update versions)
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
+archivebox add --depth=1 'https://news.ycombinator.com'
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage">Usage: CLI Usage</a> wiki. ➡️</i>
 </details>
@@ -547,11 +526,10 @@ archivebox help              # get list of archivebox subcommands that can be ru
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
 # docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox init
-docker compose run archivebox install
+docker compose run archivebox init --setup
 docker compose run archivebox version
 docker compose run archivebox help
-docker compose run archivebox add 'https://example.com'
+docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
 # to start webserver: docker compose up
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@@ -565,13 +543,12 @@ docker compose run archivebox add 'https://example.com'
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you create and cd into in a new empty directory first  
 <br/>
-# docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox:dev init
-docker run -v $PWD:/data -it archivebox/archivebox:dev install
-docker run -v $PWD:/data -it archivebox/archivebox:dev version
-docker run -v $PWD:/data -it archivebox/archivebox:dev help
-docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
-# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
+# docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
+docker run -v $PWD:/data -it archivebox/archivebox init --setup
+docker run -v $PWD:/data -it archivebox/archivebox version
+docker run -v $PWD:/data -it archivebox/archivebox help
+docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.ycombinator.com'
+# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
 </details>
@@ -603,12 +580,11 @@ nano docker-compose.yml                        # setup initial ADMIN_USERNAME &
 docker compose up                              # start the server
 <br/>
 # Or with a Docker container:
-docker run -v $PWD:/data -it archivebox/archivebox:dev archivebox manage createsuperuser
-docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
+docker run -v $PWD:/data -it archivebox/archivebox archivebox manage createsuperuser
+docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>

-<sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
-<sup>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
+<sup>Open <a href="http://localhost:8000"><code>http://localhost:8000</code></a> to see your server's Web UI ➡️</sup>
 <br/><br/>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Usage: Web UI</a> wiki. ➡️</i>
 <br/><br/>
@@ -630,15 +606,15 @@ docker compose run archivebox config --set ...

 > [!TIP]
 > Whether in Docker or not, ArchiveBox commands work the same way, and can be used to access the same data on-disk.
-> For example, you could run the Web UI in Docker Compose, and run one-off commands with `uv`-installed ArchiveBox.
+> For example, you could run the Web UI in Docker Compose, and run one-off commands with `pip`-installed ArchiveBox.

 <details>
 <summary><i>Expand to show comparison...</i></summary><br/>

 <pre lang="bash"><code style="white-space: pre-line">
-archivebox add --depth=1 'https://example.com'                     # add a URL with uv-installed archivebox on the host
+archivebox add --depth=1 'https://example.com'                     # add a URL with pip-installed archivebox on the host
 docker compose run archivebox add --depth=1 'https://example.com'                       # or w/ Docker Compose
-docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://example.com'  # or w/ Docker, all equivalent
+docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://example.com'  # or w/ Docker, all equivalent
 </code></pre>

 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">Docker</a> wiki. ➡️</i>
@@ -696,62 +672,23 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/e1e5bd78-b0b6-45dc-914c-e1046fee4bc4" width="330px" align="right" style="float: right"/>


-<!--
-```bash
-set -euo pipefail
-__archivebox_docs_home="$(mktemp -d)"
-export HOME="$__archivebox_docs_home"
-mkdir -p ~/archivebox/data ~/Downloads
-cd ~/archivebox/data
-archivebox init
-cat > ~/Downloads/some_feed.xml <<'EOF'
-<?xml version="1.0" encoding="UTF-8"?>
-<rss version="2.0">
-  <channel>
-    <title>ArchiveBox docs test feed</title>
-    <link>https://example.com/</link>
-    <description>ArchiveBox docs test feed</description>
-    <item>
-      <title>Feed item</title>
-      <link>https://example.com/from-feed</link>
-      <guid>https://example.com/from-feed</guid>
-    </item>
-  </channel>
-</rss>
-EOF
-```
-->
-<!--pytest-codeblocks:cont-->
 ```bash
 # archivebox add --help
-archivebox add --plugins=parse_txt_urls 'https://example.com/some/page'
-archivebox add --depth=1 --plugins=parse_rss_urls < "$HOME/Downloads/some_feed.xml"
-archivebox add --plugins=parse_txt_urls 'https://example.com/docs-example'
-echo 'http://example.com' | archivebox add --plugins=parse_txt_urls
-echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add --plugins=parse_txt_urls
+archivebox add 'https://example.com/some/page'
+archivebox add --parser=generic_rss < ~/Downloads/some_feed.xml
+archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
+echo 'http://example.com' | archivebox add
+echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add

 # if using Docker, add -i when piping stdin:
-# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox:dev add
+# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox add
 # if using Docker Compose, add -T when piping stdin / stdout:
 # echo 'https://example.com' | docker compose run -T archivebox add
 ```
-<!--pytest-codeblocks:cont-->
-<!--
-```bash
-archivebox list --json > snapshots.json
-grep -q 'https://example.com/some/page' snapshots.json
-grep -q 'https://example.com/from-feed' snapshots.json
-grep -q 'https://example.com/docs-example' snapshots.json
-grep -q 'http://example.com' snapshots.json
-test -d archive/users/system/crawls
-test "$(grep -c '\"url\"' snapshots.json)" -ge 4
-rm -rf "$__archivebox_docs_home"
-```
-->

 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.

-It also includes a built-in scheduled import feature with `archivebox schedule`, handled by the same orchestrator that powers `archivebox server`, so you can pull in URLs from RSS feeds and websites regularly without a separate cron container.
+It also includes a built-in scheduled import feature with `archivebox schedule` and browser bookmarklet, so you can pull in URLs from RSS feeds, websites, or the filesystem regularly/on-demand.

 <br/>

@@ -812,7 +749,7 @@ env CHROME_BINARY=chromium archivebox ...       # run with a one-off config
 <sub>These methods also work the same way when run inside Docker, see the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Docker Configuration</a> wiki page for details.</sub>
 </details><br/>

-The configuration is documented here: **[Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)**, and loaded from: [`archivebox/config/`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config/).
+The configuration is documented here: **[Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)**, and loaded here: [`archivebox/config.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config.py).

 <a name="most-common-options-to-tweak"></a>
 <details>
@@ -823,12 +760,16 @@ The configuration is documented here: **[Configuration Wiki](https://github.com/
 <br/>
 TIMEOUT=240                # default: 60    add more seconds on slower networks
 CHECK_SSL_VALIDITY=False   # default: True  False = allow saving URLs w/ bad SSL
+SAVE_ARCHIVE_DOT_ORG=False # default: True  False = disable Archive.org saving
+MAX_MEDIA_SIZE=1500m       # default: 750m  raise/lower youtubedl output size
 <br/>
 PUBLIC_INDEX=True          # default: True  whether anon users can view index
 PUBLIC_SNAPSHOTS=True      # default: True  whether anon users can view pages
 PUBLIC_ADD_VIEW=False      # default: False whether anon users can add new URLs
 <br/>
-USER_AGENT="Mozilla/5.0 ..."  # change this to get around bot blocking
+CHROME_USER_AGENT="Mozilla/5.0 ..."  # change these to get around bot blocking
+WGET_USER_AGENT="Mozilla/5.0 ..."
+CURL_USER_AGENT="Mozilla/5.0 ..."
 </code></pre>
 </details>
 <br/>
@@ -850,14 +791,14 @@ ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.c
 </blockquote>

 <ul>
-<li>Language: Python <code>&gt;=3.13</code></li>
+<li>Language: Python <code>&gt;=3.10</code></li>
 <li>Backend: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://django-ninja.dev/">Django-Ninja</a> for REST API</li>
-<li>Frontend: <a href="https://docs.djangoproject.com/en/6.0/ref/contrib/admin/">Django Admin</a> + Vanilla HTML, CSS, JS</li>
-<li>Web Server: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://github.com/django/daphne/"><code>daphne</code></a> (ASGI)</li>
-<li>Database: <a href="https://docs.djangoproject.com/en/6.0/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite3</code></li>
-<li>Job Queue: Custom orchestrator using <code>supervisord</code> for worker management</li>
-<li>Build/test/lint: <a href="https://github.com/astral-sh/uv"><code>uv</code></a> / <code>pyright</code>+<code>ty</code>+<code>pytest</code> / <code>ruff</code></li>
-<li>Subdependencies: <a href="https://github.com/ArchiveBox/abxpkg"><code>abxpkg</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
+<li>Frontend: <a href="https://docs.djangoproject.com/en/5.1/ref/contrib/admin/">Django Admin</a> + Vanilla HTML, CSS, JS</li>
+<li>Web Server: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://channels.readthedocs.io/en/latest/"><code>channels</code></a> + <a href="https://github.com/django/daphne/"><code>daphne]</code></a></li>
+<li>Database: <a href="https://docs.djangoproject.com/en/5.1/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite</code></li>
+<li>Job Queue: <a href="https://huey.readthedocs.io/">Huey</a> using <code>./data/queue.sqlite3</code> under <code>supervisord</code></li>
+<li>Build/test/lint: <a href="https://github.com/pdm-project/pdm"><code>pdm</code></a> / <code>mypy</code>+<code>pyright</code>+<code>pytest</code> / <code>ruff</code></li>
+<li>Subdependencies: <a href="https://github.com/ArchiveBox/pydantic-pkgr"><code>pydantic-pkgr</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
 </ul>


@@ -882,9 +823,11 @@ You don't need to install every dependency to use ArchiveBox. ArchiveBox will au
  
 If not using Docker, make sure to keep the dependencies up-to-date yourself and check that ArchiveBox isn't reporting any incompatibility with the versions you install.

-<pre lang="bash"><code style="white-space: pre-line"># install uv + archivebox first (see Quickstart instructions above)
+<pre lang="bash"><code style="white-space: pre-line">#install python3 and archivebox with your system package manager
+# apt/brew/pip/etc install ... (see Quickstart instructions above)
 <br/>
-archivebox install     # auto install all the extractors and extras
+which -a archivebox    # see where you have installed archivebox
+archivebox setup       # auto install all the extractors and extras
 archivebox --version   # see info and check validity of installed dependencies
 </code></pre>
  
@@ -954,7 +897,7 @@ Each snapshot subfolder <code>data/archive/TIMESTAMP/</code> includes a static <

 ## Static Archive Exporting

-You can export your index as static HTML using `archivebox list` (so you can view it without an ArchiveBox server).
+You can create one-off archives of individual URLs with `archivebox oneshot`, or export your index as static HTML using `archivebox list` (so you can view it without an ArchiveBox server).

 <br/>
 <details>
@@ -964,7 +907,10 @@ You can export your index as static HTML using `archivebox list` (so you can vie
 <p><em>NOTE: These exports are not paginated, exporting many URLs or the entire archive at once may be slow. Use the filtering CLI flags on the <code>archivebox list</code> command to export specific Snapshots or ranges.</em></p>
 </blockquote>

-<pre lang="bash"><code style="white-space: pre-line"># archivebox list --help
+<pre lang="bash"><code style="white-space: pre-line"># do a one-off single URL archive wihout needing a data dir initialized
+archivebox oneshot 'https://example.com'
+
+# archivebox list --help
 archivebox list --html --with-headers > index.html     # export to static html table
 archivebox list --json --with-headers > index.json     # export to json blob
 archivebox list --csv=timestamp,url,title > index.csv  # export to csv spreadsheet
@@ -1009,11 +955,18 @@ If you're importing pages with private content or URLs containing secret tokens
 archivebox add 'https://docs.google.com/document/d/12345somePrivateDocument'
 archivebox add 'https://vimeo.com/somePrivateVideo'

+# without first disabling saving to Archive.org:
+archivebox config --set SAVE_ARCHIVE_DOT_ORG=False  # disable saving all URLs in Archive.org
+
 # restrict the main index, Snapshot content, and Add Page to authenticated users as-needed:
 archivebox config --set PUBLIC_INDEX=False
 archivebox config --set PUBLIC_SNAPSHOTS=False
-archivebox config --set PUBLIC_ADD_VIEW=False
+archivebox config --set PUBLIC_ADD_VIEW=False 
 archivebox manage createsuperuser
+
+# if extra paranoid or anti-Google:
+archivebox config --set SAVE_FAVICON=False          # disable favicon fetching (it calls a Google API passing the URL's domain part only)
+archivebox config --set CHROME_BINARY=chromium      # ensure it's using Chromium instead of Chrome
 </code></pre>

 <blockquote>
@@ -1136,13 +1089,13 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <br/>

 <ul>
-<li><strong>ArchiveBox can use anywhere from ~1gb per 1000 Snapshots, to ~50gb per 1000 Snapshots</strong>, mostly dependent on whether you're saving video/audio using <code>YTDLP_ENABLED=True</code> and whether you lower <code>YTDLP_MAX_SIZE=750m</code>.</li>
+<li><strong>ArchiveBox can use anywhere from ~1gb per 1000 Snapshots, to ~50gb per 1000 Snapshots</strong>, mostly dependent on whether you're saving audio & video using <code>SAVE_MEDIA=True</code> and whether you lower <code>MEDIA_MAX_SIZE=750mb</code>.</li>
 <li>Disk usage can be reduced by using a compressed/<a href="https://www.ixsystems.com/blog/ixsystems-and-klara-systems-celebrate-valentines-day-with-a-heartfelt-donation-of-fast-dedupe-to-openzfs-and-truenas/">deduplicated</a> filesystem like <a href="https://www.reddit.com/r/zfs/comments/t9cexx/a_simple_real_world_zfs_compression_speed_an/">ZFS</a>/BTRFS, or by turning off extractors methods you don't need. You can also deduplicate content with a tool like <a href="https://github.com/adrianlopezroche/fdupes"><code>fdupes</code></a> or <a href="https://github.com/pauldreik/rdfind"><code>rdfind</code></a>.  
 </li>
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
 </li>
 <li><strong>Try to keep the <code>data/index.sqlite3</code> file on local drive (not a network mount)</strong> or SSD for maximum performance, however the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
-<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, make sure the mounted data directory is writable by its intended owner and consider disabling <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304"><code>root_squash</code></a> on your fileshare server.
+<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, you may need to set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid"><code>PUID</code> & <code>PGID</code></a> and <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304">disable <code>root_squash</code></a> on your fileshare server.
 </li>
 </ul>

@@ -1152,6 +1105,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid">Wiki: Configuration (<code>PUID</code> & <code>GUID</code>)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root">Wiki: Security Overview (Do Not Run as Root)</a></li>
 </ul>

@@ -1241,7 +1195,7 @@ ArchiveBox's stance is that duplication of other people's content is only ethica

 In the U.S., <a href="https://guides.library.oregonstate.edu/copyright/libraries">libraries, researchers, and archivists</a> are allowed to duplicate copyrighted materials under <a href="https://libguides.ala.org/copyright/fairuse">"fair use"</a> for <a href="https://guides.cuny.edu/cunyfairuse/librarians#:~:text=One%20of%20these%20specified%20conditions,may%20be%20liable%20for%20copyright">private study, scholarship, or research</a>. Archive.org's non-profit preservation work is <a href="https://blog.archive.org/2024/03/01/fair-use-in-action-at-the-internet-archive/">covered under fair use</a> in the US, and they properly handle <a href="https://cardozoaelj.com/2015/03/20/use-of-copyright-law-to-take-down-revenge-porn/">unethical content</a>/<a href="https://help.archive.org/help/rights/">DMCA</a>/<a href="https://gdpr.eu/right-to-be-forgotten/#:~:text=An%20individual%20has%20the%20right,that%20individual%20withdraws%20their%20consent.">GDPR</a> removal requests to maintain good standing in the eyes of the law.

-As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use software like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal council if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and changing your instance's branding using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir"><code>CUSTOM_TEMPLATES_DIR</code></a>).
+As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use sofware like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal council if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and changing your instance's branding using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir"><code>CUSTOM_TEMPLATES_DIR</code></a>).

 </details>
 <br/>
@@ -1270,7 +1224,7 @@ ArchiveBox tries to be a robust, set-and-forget archiving solution suitable for

 Not all content is suitable to be archived on a centralized, publicly accessible platform. Archive.org doesn't offer the ability to save things behind login walls for good reason, as the content may not have been intended for a public audience. ArchiveBox exists to fill that gap by letting everyone save what they have access to on an individual basis, and to encourage decentralized archiving that's less succeptible to censorship or natural disasters.

-By having users store their content locally or within their organizations, we can also save much larger portions of the internet than a centralized service has the disk capacity to handle. The eventual goal is to work towards federated archiving where users can share portions of their collections with each other, and with central archives on a case-by-case basis.
+By having users store their content locally or within their organizations, we can also save much larger portions of the internet than a centralized service has the disk capcity handle. The eventual goal is to work towards federated archiving where users can share portions of their collections with each other, and with central archives on a case-by-case basis.

 <h3>Comparison With Other Self-Hosted Archiving Options</h3>

@@ -1279,7 +1233,7 @@ ArchiveBox differentiates itself from [similar self-hosted projects](https://git

 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*

-*If you want more bookmark categorization and note-taking features, check out [Memex](https://github.com/WorldBrain/Memex), [Hoarder](https://github.com/hoarder-app/hoarder), [LinkWarden](https://github.com/linkwarden/linkwarden),  [Archivy](https://archivy.github.io/), or [LinkAce](https://www.linkace.org/).*
+*If you want more bookmark categorization and note-taking features, check out [Archivy](https://archivy.github.io/), [Memex](https://github.com/WorldBrain/Memex), [Polar](https://getpolarized.io/), or [LinkAce](https://www.linkace.org/).*

 *If you need more advanced recursive spider/crawling ability beyond `--depth=1`, check out [Browsertrix](https://github.com/webrecorder/browsertrix-crawler), [Photon](https://github.com/s0md3v/Photon), or [Scrapy](https://scrapy.org/) and pipe the outputted URLs into ArchiveBox.*

@@ -1395,65 +1349,49 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc

 **Internal Architecture Diagrams:** https://github.com/ArchiveBox/ArchiveBox/wiki/ArchiveBox-Architecture-Diagrams

+<img width="100%" alt="architecture diagrams" src="https://github.com/user-attachments/assets/bdf782d5-2c8f-4168-845d-6794ff5cbfec">
+

 ### Setup the dev environment

 <details><summary><i>Click to expand...</i></summary>

-#### 1. Setup the monorepo
+#### 1. Clone the main code repo (making sure to pull the submodules as well)

-First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/
-
-```console
-git clone https://github.com/ArchiveBox/monorepo
-cd monorepo
-./bin/setup.sh
-
-# activate the monorepo venv, then cd into archivebox
-source .venv/bin/activate
-cd archivebox
+```bash
+git clone --recurse-submodules https://github.com/ArchiveBox/ArchiveBox
+cd ArchiveBox
+git checkout dev  # or the branch you want to test
+git submodule update --init --recursive
+git pull --recurse-submodules
 ```

-Repos included in monorepo setup:
-
- https://github.com/ArchiveBox/abxbus
- https://github.com/ArchiveBox/abxpkg
- https://github.com/ArchiveBox/abx-plugins
- https://github.com/ArchiveBox/abx-dl
- https://github.com/ArchiveBox/ArchiveBox
-
-
 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine

-```console
-# Install ArchiveBox runtime dependencies
-mkdir -p data && cd data
+```bash
+# Install ArchiveBox + python dependencies
+pip install uv
+uv venv
+uv sync
+
 archivebox init
-archivebox install         # detect and install all extractor dependencies
-
-# Run the development server w/ autoreloading (but no bg workers)
-archivebox server --debug --reload 0.0.0.0:8000
-
-# Run the production server (with bg workers but no autoreloading)
-archivebox server 0.0.0.0:8000
+archivebox setup
 ```

 #### 2. Option B: Build the docker container and use that for development instead

-```console
+```bash
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
 # inside the container will reload and pick up your changes
-./bin/build_docker.sh dev
-
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev init
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev install
-
-# Run the development server w/ autoreloading (but no bg workers)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
-
-# Run the production server (with bg workers but no autoreloading)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server
+docker build . -t archivebox
+docker run -it \
+    -v $PWD/data:/data \
+    archivebox init --setup
+docker run -it -p 8000:8000 \
+    -v $PWD/data:/data \
+    -v $PWD/archivebox:/app/archivebox \
+    archivebox server 0.0.0.0:8000 --debug --reload

 # (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
 # When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@@ -1470,16 +1408,12 @@ You can also run all these in Docker. For more examples see the GitHub Actions C

 <details><summary><i>Click to expand...</i></summary>

-```console
-# set up persistent DEBUG=True for all runs
+```bash
 archivebox config --set DEBUG=True
-
-# OR you can run a dev server with DEBUG=True in a few ways:
-archivebox server --debug --reload 0.0.0.0:8000
 # or
-archivebox server --debug 0.0.0.0:8000
-# or
-env DEBUG=True daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
+archivebox server --debug ...
+# faster dev version wo/ bg workers enabled:
+daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
 ```

 https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
@@ -1494,17 +1428,17 @@ https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-dj

 If you're looking for the latest `dev` Docker image, it's often available pre-built on Docker Hub, simply pull and use `archivebox/archivebox:dev`.

-```console
+```bash
 docker pull archivebox/archivebox:dev
 docker run archivebox/archivebox:dev version
 # verify the BUILD_TIME and COMMIT_HASH in the output are recent
 ```

 ##### Build Branch from Source
-
+  
 You can also build and run any branch yourself from source, for example to build & use `dev` locally:

-```console
+```bash
 # docker-compose.yml:
 services:
    archivebox:
@@ -1528,11 +1462,11 @@ archivebox install

 <details><summary><i>Click to expand...</i></summary>

-```console
+```bash
 ./bin/lint.sh
 ./bin/test.sh
 ```
-(uses `ruff`, `pyright`, `ty`, and `pytest -s`)
+(uses `flake8`, `mypy`, and `pytest -s`)

 </details>

@@ -1541,7 +1475,7 @@ archivebox install

 <details><summary><i>Click to expand...</i></summary>

-```console
+```bash
 # generate the database migrations after changes to models.py
 cd archivebox/
 ./manage.py makemigrations
@@ -1576,9 +1510,13 @@ archivebox manage runscript testdata
 archivebox manage reset_db

 # use django-tui to interactively explore commands
-uv pip install django-tui
+pip install django-tui
 # ensure django-tui is in INSTALLED_APPS: core/settings.py
 archivebox manage tui
+
+# show python and JS package dependency trees
+pdm list --tree
+npm ls --all
 ```

 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/dc3e9f8c-9544-46e0-a7f0-30f571b72022" width="600px" alt="ArchiveBox ORM models relatinoship graph"/>
@@ -1594,18 +1532,47 @@ archivebox manage tui

 #### Contributing a new extractor

+<details><summary><i>Click to expand...</i></summary>

-Extractors are maintained in a separate repo here: https://github.com/ArchiveBox/abx-plugins (included in monorepo setup).
+<br/><br/>

-Copy a similar plugin as a template to modify, then open a new PR to add it in that repo.
+ArchiveBox [`extractors`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/media.py) are external binaries or Python/Node scripts that ArchiveBox runs to archive content on a page.

+Extractors take the URL of a page to archive, write their output to the filesystem `data/archive/TIMESTAMP/EXTRACTOR/...`, and return an [`ArchiveResult`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/models.py#:~:text=return%20qs-,class%20ArchiveResult,-(models.Model)%3A) entry which is saved to the database (visible on the `Log` page in the UI).
+
+*Check out how we added **[`archivebox/extractors/singlefile.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/singlefile.py)** as an example of the process: [Issue #399](https://github.com/ArchiveBox/ArchiveBox/issues/399) + [PR #403](https://github.com/ArchiveBox/ArchiveBox/pull/403).*
+
+<br/>
+
+
+**The process to contribute a new extractor is like this:**
+
+> [!IMPORTANT]
+> This process is getting much easier after v0.8.x, there is a new plugin system under development: https://github.com/ArchiveBox/ArchiveBox/releases/tag/v0.8.4-rc
+
+1. [Open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new?assignees=&labels=changes%3A+behavior%2Cstatus%3A+idea+phase&template=feature_request.md&title=Feature+Request%3A+...) with your propsoed implementation (please link to the pages of any new external dependencies you plan on using)
+2. Ensure any dependencies needed are easily installable via a package managers like `apt`, `brew`, `pip3`, `npm`
+   (Ideally, prefer to use external programs available via `pip3` or `npm`, however we do support using any binary installable via package manager that exposes a CLI/Python API and writes output to stdout or the filesystem.)
+3. Create a new file in [`archivebox/extractors/EXTRACTOR.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors) (copy an existing extractor like [`singlefile.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/singlefile.py) as a template)
+4. Add config settings to enable/disable any new dependencies and the extractor as a whole, e.g. `USE_DEPENDENCYNAME`, `SAVE_EXTRACTORNAME`, `EXTRACTORNAME_SOMEOTHEROPTION` in [`archivebox/config.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config.py)
+5. Add a preview section to [`archivebox/templates/core/snapshot.html`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) to view the output, and a column to [`archivebox/templates/core/index_row.html`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/index_row.html) with an icon for your extractor
+6. Add an integration test for your extractor in [`tests/test_extractors.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/tests/test_extractors.py)
+7. [Submit your PR for review!](https://github.com/ArchiveBox/ArchiveBox/blob/dev/.github/CONTRIBUTING.md) 🎉
+8. Once merged, please document it in these places and anywhere else you see info about other extractors:
+  - https://github.com/ArchiveBox/ArchiveBox#output-formats
+  - https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles
+  - https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies
+
+<br/><br/>
+
+</details>

 #### Build the docs, pip package, and docker image

 <details><summary><i>Click to expand...</i></summary>

 (Normally CI takes care of this, but these scripts can be run to do it manually)
-```console
+```bash
 ./bin/build.sh

 # or individually:
@@ -1621,7 +1588,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <details><summary><i>Click to expand...</i></summary>

 (Normally CI takes care of this, but these scripts can be run to do it manually)
-```console
+```bash
 ./bin/release.sh

 # or individually:
@@ -1653,8 +1620,7 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <a href="https://github.com/sponsors/pirate"><img src="https://img.shields.io/badge/Github_Sponsors-%23B7CDFE.svg"/></a> &nbsp;
 <a href="https://www.patreon.com/theSquashSH"><img src="https://img.shields.io/badge/Patreon-%23DD5D76.svg"/></a> &nbsp;
 <a href="https://paypal.me/NicholasSweeting"><img src="https://img.shields.io/badge/Paypal-%23FFD141.svg"/></a> &nbsp;
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a> &nbsp;
-<a href="https://archivebox-shop.fourthwall.com/"><img src="https://img.shields.io/badge/Merch-%23903851.svg"/></a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a>
 <br/>
 <sup><i>ArchiveBox operates as a US 501(c)(3) nonprofit <a href="https://en.wikipedia.org/wiki/Fiscal_sponsorship">FSP</a> (sponsored by <a href="https://hackclub.com/hcb?ref=donation">HCB</a>), <a href="https://hcb.hackclub.com/donations/start/archivebox">direct donations</a> are tax-deductible.</i></sup>
 <br/><br/>
@@ -45,7 +45,7 @@
 ### Crawls App

 - Archive an entire website -> [Crawl page]
-    - What are the starting URLs?
+    - What are the seed URLs?
    - How many hops to follow?
    - Follow links to external domains?
    - Follow links to parent URLs?
@@ -169,4 +169,4 @@ class AlertDestination(models.Model):
    maximum_frequency
    filter_rules
    credentials
-    alert_template: JINJA2 json/text template that gets populated with alert contents
+    alert_template: JINJA2 json/text template that gets populated with alert contents
@@ -9,23 +9,12 @@
 # in a universe that seems indifferent to us."
 # --Norber Weiner

-__package__ = "archivebox"
+__package__ = 'archivebox'

 import os
 import sys
 from pathlib import Path
-from typing import Protocol, cast
-
-
-class _ReconfigurableStream(Protocol):
-    def reconfigure(self, *, line_buffering: bool) -> object: ...
-
-
-# Force unbuffered output for real-time logs
-if hasattr(sys.stdout, "reconfigure"):
-    cast(_ReconfigurableStream, sys.stdout).reconfigure(line_buffering=True)
-    cast(_ReconfigurableStream, sys.stderr).reconfigure(line_buffering=True)
-os.environ["PYTHONUNBUFFERED"] = "1"
+from typing import cast

 ASCII_LOGO = """
 █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
@@ -36,79 +25,83 @@ ASCII_LOGO = """
 ╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
 """

+# make sure PACKAGE_DIR is in sys.path so we can import all subfolders
+# without necessarily waiting for django to load them thorugh INSTALLED_APPS
 PACKAGE_DIR = Path(__file__).resolve().parent
-
-# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models
-# # Migrations reference models like 'machine.Binary' which need to be importable
-# if str(PACKAGE_DIR) not in sys.path:
-#     sys.path.append(str(PACKAGE_DIR))
-
-os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
-os.environ["TZ"] = "UTC"
+if str(PACKAGE_DIR) not in sys.path:
+    sys.path.append(str(PACKAGE_DIR))
+os.environ['DJANGO_SETTINGS_MODULE'] = 'core.settings'
+os.environ['TZ'] = 'UTC'

 # detect ArchiveBox user's UID/GID based on data dir ownership
-from .config.permissions import drop_privileges  # noqa
-
+from .config.permissions import drop_privileges                 # noqa
 drop_privileges()

-from .misc.checks import check_not_root, check_not_inside_source_dir, check_io_encoding  # noqa
-
+from .misc.checks import check_not_root, check_io_encoding      # noqa
 check_not_root()
-check_not_inside_source_dir()
 check_io_encoding()

-from .config.version import VERSION  # noqa
+# print('INSTALLING MONKEY PATCHES')
+from .monkey_patches import *                    # noqa
+# print('DONE INSTALLING MONKEY PATCHES')


+# print('LOADING VENDORED LIBRARIES')
+from .pkgs import load_vendored_pkgs             # noqa
+load_vendored_pkgs()
+# print('DONE LOADING VENDORED LIBRARIES')
+
+# Load ABX Plugin Specifications + Default Implementations
+import abx                                       # noqa
+import abx_spec_archivebox                       # noqa
+import abx_spec_config                           # noqa
+import abx_spec_pydantic_pkgr                    # noqa
+import abx_spec_django                           # noqa
+import abx_spec_searchbackend                    # noqa
+
+abx.pm.add_hookspecs(abx_spec_config.PLUGIN_SPEC)
+abx.pm.register(abx_spec_config.PLUGIN_SPEC())
+
+abx.pm.add_hookspecs(abx_spec_pydantic_pkgr.PLUGIN_SPEC)
+abx.pm.register(abx_spec_pydantic_pkgr.PLUGIN_SPEC())
+
+abx.pm.add_hookspecs(abx_spec_django.PLUGIN_SPEC)
+abx.pm.register(abx_spec_django.PLUGIN_SPEC())
+
+abx.pm.add_hookspecs(abx_spec_searchbackend.PLUGIN_SPEC)
+abx.pm.register(abx_spec_searchbackend.PLUGIN_SPEC())
+
+# Cast to ArchiveBoxPluginSpec to enable static type checking of pm.hook.call() methods
+abx.pm = cast(abx.ABXPluginManager[abx_spec_archivebox.ArchiveBoxPluginSpec], abx.pm)
+pm = abx.pm
+
+
+# Load all pip-installed ABX-compatible plugins
+ABX_ECOSYSTEM_PLUGINS = abx.get_pip_installed_plugins(group='abx')
+
+# Load all built-in ArchiveBox plugins
+ARCHIVEBOX_BUILTIN_PLUGINS = {
+    'config': PACKAGE_DIR / 'config',
+    'core': PACKAGE_DIR / 'core',
+    # 'search': PACKAGE_DIR / 'search',
+    # 'core': PACKAGE_DIR / 'core',
+}
+
+# Load all user-defined ArchiveBox plugins
+USER_PLUGINS = abx.find_plugins_in_dir(Path(os.getcwd()) / 'user_plugins')
+
+# Import all plugins and register them with ABX Plugin Manager
+ALL_PLUGINS = {**ABX_ECOSYSTEM_PLUGINS, **ARCHIVEBOX_BUILTIN_PLUGINS, **USER_PLUGINS}
+LOADED_PLUGINS = abx.load_plugins(ALL_PLUGINS)
+
+# Setup basic config, constants, paths, and version
+from .config.constants import CONSTANTS                         # noqa
+from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR    # noqa
+from .config.version import VERSION                             # noqa
+
 __version__ = VERSION
-__author__ = "ArchiveBox"
-__license__ = "MIT"
-
-
-def __getattr__(name: str):
-    if name == "CONSTANTS":
-        from .config.constants import CONSTANTS
-
-        os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
-        return CONSTANTS
-    if name == "DATA_DIR":
-        from .config.paths import DATA_DIR
-
-        return DATA_DIR
-    if name == "VERSION":
-        return VERSION
-    if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
-        from abx_plugins import get_plugins_dir
-        from .config.constants import CONSTANTS
-
-        builtin_plugins_dir = Path(get_plugins_dir()).resolve()
-        user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
-        plugins = {
-            "builtin": builtin_plugins_dir,
-            "user": user_plugins_dir,
-        }
-        values = {
-            "BUILTIN_PLUGINS_DIR": builtin_plugins_dir,
-            "USER_PLUGINS_DIR": user_plugins_dir,
-            "ALL_PLUGINS": plugins,
-            "LOADED_PLUGINS": plugins,
-        }
-        return values[name]
-    raise AttributeError(name)
-
-
-__all__ = (
-    "ASCII_LOGO",
-    "ASCII_ICON",
-    "PACKAGE_DIR",
-    "DATA_DIR",
-    "CONSTANTS",
-    "VERSION",
-    "BUILTIN_PLUGINS_DIR",
-    "USER_PLUGINS_DIR",
-    "ALL_PLUGINS",
-    "LOADED_PLUGINS",
-)
+__author__ = 'ArchiveBox'
+__license__ = 'MIT'

 ASCII_ICON = """
 ██████████████████████████████████████████████████████████████████████████████████████████████████ 
@@ -1,20 +1,19 @@
 #!/usr/bin/env python3
-"""This is the entrypoint for python -m archivebox ..."""
+"""This is the main entry point for the ArchiveBox CLI."""
+__package__ = 'archivebox'

-__package__ = "archivebox"
-
-import archivebox  # noqa # make sure monkey patches are applied before anything else
+import archivebox      # noqa # make sure monkey patches are applied before anything else
 import sys

 from .cli import main

 ASCII_LOGO_MINI = r"""
-     _             _     _           ____
+     _             _     _           ____            
    / \   _ __ ___| |__ (_)_   _____| __ )  _____  __
   / _ \ | '__/ __| '_ \| \ \ / / _ \  _ \ / _ \ \/ /
-  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  <
+  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  < 
 /_/   \_\_|  \___|_| |_|_| \_/ \___|____/ \___/_/\_\
 """

-if __name__ == "__main__":
-    main(args=sys.argv[1:])
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -0,0 +1 @@
+__package__ = 'abid_utils'
@@ -0,0 +1,223 @@
+__package__ = 'archivebox.abid_utils'
+
+from typing import NamedTuple, Any, Union, Dict
+
+import ulid
+import uuid6
+import hashlib
+from urllib.parse import urlparse
+
+from uuid import UUID
+from typeid import TypeID            # type: ignore[import-untyped]
+from datetime import datetime
+
+from archivebox.misc.util import enforce_types
+
+
+ABID_PREFIX_LEN = 4
+ABID_SUFFIX_LEN = 26
+ABID_LEN = 30
+ABID_TS_LEN = 10
+ABID_URI_LEN = 8
+ABID_SUBTYPE_LEN = 2
+ABID_RAND_LEN = 6
+
+DEFAULT_ABID_PREFIX = 'obj_'
+
+# allows people to keep their uris secret on a per-instance basis by changing the salt.
+# the default means everyone can share the same namespace for URI hashes,
+# meaning anyone who has a URI and wants to check if you have it can guess the ABID
+DEFAULT_ABID_URI_SALT = '687c2fff14e3a7780faa5a40c237b19b5b51b089'
+
+
+class ABID(NamedTuple):
+    """
+    e.g. ABID('obj_01HX9FPYTRE4A5CCD901ZYEBQE')
+    """
+    prefix: str            # e.g. obj_
+    ts: str                # e.g. 01HX9FPYTR
+    uri: str               # e.g. E4A5CCD9
+    subtype: str           # e.g. 01
+    rand: str              # e.g. ZYEBQE
+    
+    # salt: str = DEFAULT_ABID_URI_SALT
+
+    def __getattr__(self, attr: str) -> Any:
+        return getattr(self.ulid, attr)
+
+    def __eq__(self, other: Any) -> bool:
+        try:
+            return self.ulid == other.ulid
+        except AttributeError:
+            return NotImplemented
+
+    def __str__(self) -> str:
+        return self.prefix + self.suffix
+
+    def __len__(self) -> int:
+        return len(self.prefix + self.suffix)
+
+    @classmethod
+    def parse(cls, buffer: Union[str, UUID, ulid.ULID, TypeID, 'ABID'], prefix=DEFAULT_ABID_PREFIX) -> 'ABID':
+        assert buffer, f'Attempted to create ABID from null value {buffer}'
+
+        buffer = str(buffer)
+        if '_' in buffer:
+            prefix, suffix = buffer.split('_')
+        else:
+            prefix, suffix = prefix.strip('_'), buffer
+
+        assert len(prefix) == ABID_PREFIX_LEN - 1   # length without trailing _
+        assert len(suffix) == ABID_SUFFIX_LEN, f'Suffix {suffix} from {buffer} was not {ABID_SUFFIX_LEN} chars long'
+
+        return cls(
+            prefix=abid_part_from_prefix(prefix),
+            ts=suffix[0:10].upper(),
+            uri=suffix[10:18].upper(),
+            subtype=suffix[18:20].upper(),
+            rand=suffix[20:26].upper(),
+        )
+    
+    @property
+    def uri_salt(self) -> str:
+        return DEFAULT_ABID_URI_SALT
+
+    @property
+    def suffix(self):
+        return ''.join((self.ts, self.uri, self.subtype, self.rand))
+    
+    @property
+    def ulid(self) -> ulid.ULID:
+        return ulid.parse(self.suffix)
+
+    @property
+    def uuid(self) -> UUID:
+        return self.ulid.uuid
+
+    @property
+    def uuid6(self) -> uuid6.UUID:
+        return uuid6.UUID(hex=self.uuid.hex)
+
+    @property
+    def typeid(self) -> TypeID:
+        return TypeID.from_uuid(prefix=self.prefix.strip('_'), suffix=self.uuid6)
+
+    @property
+    def datetime(self) -> datetime:
+        return self.ulid.timestamp().datetime
+
+
+
+####################################################
+
+
+@enforce_types
+def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
+    """
+    https://example.com -> 'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25' (example.com)
+    """
+    if isinstance(uri, bytes):
+        uri_str: str = uri.decode()
+    else:
+        uri_str = str(uri)
+
+    # only hash the domain part of URLs
+    if '://' in uri_str:
+        try:
+            domain = urlparse(uri_str).netloc
+            if domain:
+                uri_str = domain
+        except AttributeError:
+            pass
+    
+    # the uri hash is the sha256 of the domain + salt
+    uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
+
+    return hashlib.sha256(uri_bytes).hexdigest().upper()
+
+@enforce_types
+def abid_part_from_prefix(prefix: str) -> str:
+    """
+    'snp_'
+    """
+    # if prefix is None:
+    #     return 'obj_'
+
+    prefix = prefix.strip('_').lower()
+    assert len(prefix) == 3
+    return prefix + '_'
+
+@enforce_types
+def abid_part_from_uri(uri: Any, salt: str=DEFAULT_ABID_URI_SALT) -> str:
+    """
+    'E4A5CCD9'     # takes first 8 characters of sha256(url)
+    """
+    uri = str(uri).strip()
+    assert uri not in ('None', '')
+    return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
+
+@enforce_types
+def abid_part_from_ts(ts: datetime) -> str:
+    """
+    '01HX9FPYTR'   # produces 10 character Timestamp section of ulid based on added date
+    """
+    return str(ulid.from_timestamp(ts))[:ABID_TS_LEN]
+
+@enforce_types
+def ts_from_abid(abid: str) -> datetime:
+    return ulid.parse(abid.split('_', 1)[-1]).timestamp().datetime
+
+@enforce_types
+def abid_part_from_subtype(subtype: str | int) -> str:
+    """
+    Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
+    Also allows us to change the ulid spec later by putting special sigil values here.
+    """
+    subtype = str(subtype)
+    if len(subtype) == ABID_SUBTYPE_LEN:
+        return subtype
+
+    return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()
+
+@enforce_types
+def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
+    """
+    'ZYEBQE'   # takes last 6 characters of randomness from existing legacy uuid db field
+    """
+    if rand is None:
+        # if it's None we generate a new random 6 character hex string
+        return str(ulid.new())[-ABID_RAND_LEN:]
+    elif isinstance(rand, UUID):
+        # if it's a uuid we take the last 6 characters of the ULID represation of it
+        return str(ulid.from_uuid(rand))[-ABID_RAND_LEN:]
+    elif isinstance(rand, int):
+        # if it's a BigAutoInteger field we convert it from an int to a 0-padded string
+        rand_str = str(rand)[-ABID_RAND_LEN:]
+        padding_needed = ABID_RAND_LEN - len(rand_str)
+        rand_str = ('0'*padding_needed) + rand_str
+        return rand_str
+
+    # otherwise treat it as a string, take the last 6 characters of it verbatim
+    return str(rand)[-ABID_RAND_LEN:].upper()
+
+
+@enforce_types
+def abid_hashes_from_values(prefix: str, ts: datetime, uri: Any, subtype: str | int, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> Dict[str, str]:
+    return {
+        'prefix': abid_part_from_prefix(prefix),
+        'ts': abid_part_from_ts(ts),
+        'uri': abid_part_from_uri(uri, salt=salt),
+        'subtype': abid_part_from_subtype(subtype),
+        'rand': abid_part_from_rand(rand),
+        # 'salt': don't add this, salt combined with uri above to form a single hash
+    }
+
+@enforce_types
+def abid_from_values(prefix: str, ts: datetime, uri: str, subtype: str, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> ABID:
+    """
+    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+    """
+
+    abid = ABID(**abid_hashes_from_values(prefix, ts, uri, subtype, rand, salt=salt))
+    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
+    return abid
@@ -0,0 +1,174 @@
+__package__ = 'archivebox.abid_utils'
+
+from typing import Any
+
+from django.contrib import admin, messages
+from django.core.exceptions import ValidationError
+from django.utils.html import format_html
+from django.utils.safestring import mark_safe
+from django.shortcuts import redirect
+
+from django_object_actions import DjangoObjectActions, action
+
+from archivebox.misc.util import parse_date
+
+from .abid import ABID
+
+
+def highlight_diff(display_val: Any, compare_val: Any, invert: bool=False, color_same: str | None=None, color_diff: str | None=None):
+    """highlight each character in red that differs with the char at the same index in compare_val"""
+
+    display_val = str(display_val)
+    compare_val = str(compare_val)
+
+    if len(compare_val) < len(display_val):
+        compare_val += ' ' * (len(display_val) - len(compare_val))
+
+    similar_color, highlighted_color = color_same or 'inherit', color_diff or 'red'
+    if invert:
+        similar_color, highlighted_color = color_same or 'green', color_diff or 'inherit'
+
+    return mark_safe(''.join(
+        format_html('<span style="color: {};">{}</span>', highlighted_color, display_val[i])
+        if display_val[i] != compare_val[i] else
+        format_html('<span style="color: {};">{}</span>', similar_color, display_val[i])
+        for i in range(len(display_val))
+    ))
+
+def get_abid_info(self, obj, request=None):
+    from archivebox.api.auth import get_or_create_api_token
+    
+    try:
+        #abid_diff = f' != obj.ABID: {highlight_diff(obj.ABID, obj.abid)} ❌' if str(obj.ABID) != str(obj.abid) else ' == .ABID ✅'
+
+        fresh_values = obj.ABID_FRESH_VALUES
+        fresh_hashes = obj.ABID_FRESH_HASHES
+        fresh_diffs = obj.ABID_FRESH_DIFFS
+        fresh_abid = ABID(**fresh_hashes)
+        
+        fresh_abid_diff = f'❌ != &nbsp; .fresh_abid: {highlight_diff(fresh_abid, obj.ABID)}' if str(fresh_abid) != str(obj.ABID) else '✅'
+        fresh_uuid_diff = f'❌ != &nbsp; .fresh_uuid: {highlight_diff(fresh_abid.uuid, obj.ABID.uuid)}' if str(fresh_abid.uuid) != str(obj.ABID.uuid) else '✅'
+
+        id_pk_diff = f'❌ !=  .pk: {highlight_diff(obj.pk, obj.id)}' if str(obj.pk) != str(obj.id) else '✅'
+
+        fresh_ts = parse_date(fresh_values['ts']) or None
+        ts_diff = f'❌ != {highlight_diff( fresh_hashes["ts"], obj.ABID.ts)}' if  fresh_hashes["ts"] != obj.ABID.ts else '✅'
+
+        derived_uri = fresh_hashes['uri']
+        uri_diff = f'❌ != {highlight_diff(derived_uri, obj.ABID.uri)}' if derived_uri != obj.ABID.uri else '✅'
+
+        derived_subtype = fresh_hashes['subtype']
+        subtype_diff = f'❌ != {highlight_diff(derived_subtype, obj.ABID.subtype)}' if derived_subtype != obj.ABID.subtype else '✅'
+
+        derived_rand = fresh_hashes['rand']
+        rand_diff = f'❌ != {highlight_diff(derived_rand, obj.ABID.rand)}' if derived_rand != obj.ABID.rand else '✅'
+
+        return format_html(
+            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+            '''
+            <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
+            <br/><hr/>
+            <div style="opacity: 0.8">
+            &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
+            &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
+            &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</small><br/>
+            <hr/>
+            &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; {}</code> &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
+            &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; {}</code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> <span style="display:inline-block; vertical-align: -4px; width: 330px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}: <code style="user-select: all">{}</code></span><br/>
+            &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
+            &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code></code>
+            <br/><hr/>
+            <span style="color: #f375a0">{}</span> <code style="color: red"><b>{}</b></code> {}
+            </div>
+            ''',
+            obj.api_url + (f'?api_key={get_or_create_api_token(request.user)}' if request and request.user else ''), obj.api_url, obj.api_docs_url,
+            highlight_diff(obj.id, obj.ABID.uuid, invert=True), mark_safe(id_pk_diff),
+            highlight_diff(obj.ABID.uuid, obj.id, invert=True), mark_safe(fresh_uuid_diff),
+            highlight_diff(obj.abid, fresh_abid), mark_safe(fresh_abid_diff),
+            # str(fresh_abid.uuid), mark_safe(fresh_uuid_diff),
+            # str(fresh_abid), mark_safe(fresh_abid_diff),
+            highlight_diff(obj.ABID.ts,  fresh_hashes['ts']), highlight_diff(str(obj.ABID.uuid)[0:14], str(fresh_abid.uuid)[0:14]), mark_safe(ts_diff), obj.abid_ts_src, fresh_ts and fresh_ts.isoformat(),
+            highlight_diff(obj.ABID.uri, derived_uri), highlight_diff(str(obj.ABID.uuid)[14:26], str(fresh_abid.uuid)[14:26]), mark_safe(uri_diff), obj.abid_uri_src, str(fresh_values['uri']),
+            highlight_diff(obj.ABID.subtype, derived_subtype), highlight_diff(str(obj.ABID.uuid)[26:28], str(fresh_abid.uuid)[26:28]), mark_safe(subtype_diff), obj.abid_subtype_src, str(fresh_values['subtype']),
+            highlight_diff(obj.ABID.rand, derived_rand), highlight_diff(str(obj.ABID.uuid)[28:36], str(fresh_abid.uuid)[28:36]), mark_safe(rand_diff), obj.abid_rand_src, str(fresh_values['rand'])[-7:],
+            'Some values the ABID depends on have changed since the ABID was issued:' if fresh_diffs else '',
+            ", ".join(diff['abid_src'] for diff in fresh_diffs.values()),
+            '(clicking "Regenerate ABID" in the upper right will assign a new ABID, breaking any external references to the old ABID)' if fresh_diffs else '',
+        )
+    except Exception as e:
+        # import ipdb; ipdb.set_trace()
+        return str(e)
+
+
+class ABIDModelAdmin(DjangoObjectActions, admin.ModelAdmin):
+    list_display = ('created_at', 'created_by', 'abid')
+    sort_fields = ('created_at', 'created_by', 'abid')
+    readonly_fields = ('created_at', 'modified_at', 'abid_info')
+    # fields = [*readonly_fields]
+    
+    change_actions = ("regenerate_abid",)
+    # changelist_actions = ("regenerate_abid",)
+
+    def _get_obj_does_not_exist_redirect(self, request, opts, object_id):
+        try:
+            object_pk = self.model.id_from_abid(object_id)
+            return redirect(self.request.path.replace(object_id, object_pk), permanent=False)
+        except (self.model.DoesNotExist, ValidationError):
+            pass
+        return super()._get_obj_does_not_exist_redirect(request, opts, object_id)       # type: ignore
+    
+    def queryset(self, request):
+        self.request = request
+        return super().queryset(request)                                                # type: ignore
+    
+    def change_view(self, request, object_id, form_url="", extra_context=None):
+        self.request = request
+        return super().change_view(request, object_id, form_url, extra_context)
+
+    def get_form(self, request, obj=None, **kwargs):
+        self.request = request
+        form = super().get_form(request, obj, **kwargs)
+        if 'created_by' in form.base_fields:
+            form.base_fields['created_by'].initial = request.user
+            
+        if obj:
+            if obj.ABID_FRESH_DIFFS:
+                messages.warning(request, "The ABID is not in sync with the object! See the API Identifiers section below for more info...")
+
+        return form
+
+    def get_formset(self, request, formset=None, obj=None, **kwargs):
+        formset = super().get_formset(request, formset, obj, **kwargs)                  # type: ignore
+        formset.form.base_fields['created_at'].disabled = True
+        
+        return formset
+
+    def save_model(self, request, obj, form, change):
+        self.request = request
+
+        old_abid = getattr(obj, '_previous_abid', None) or obj.abid
+
+        super().save_model(request, obj, form, change)
+        obj.refresh_from_db()
+
+        new_abid = obj.abid
+        if new_abid != old_abid:
+            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
+        # import ipdb; ipdb.set_trace()
+
+    @admin.display(description='API Identifiers')
+    def abid_info(self, obj):
+        return get_abid_info(self, obj, request=self.request)
+
+    @action(label="Regenerate ABID", description="Re-Generate the ABID based on fresh values")
+    def regenerate_abid(self, request, obj):
+        old_abid = str(obj.abid)
+        obj.abid = obj.issue_new_abid(overwrite=True)
+        obj.save()
+        obj.refresh_from_db()
+        new_abid = str(obj.abid)
+
+        if new_abid != old_abid:
+            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
+        else:
+            messages.success(request, "The ABID was not regenerated, it is already up-to-date with the object.")
@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class AbidUtilsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    
+    name = 'abid_utils'
@@ -0,0 +1,509 @@
+"""
+This file provides the Django ABIDField and ABIDModel base model to inherit from.
+"""
+
+
+from typing import Any, Dict, Union, List, Set, cast
+
+from uuid import uuid4
+from functools import partial
+from charidfield import CharIDField  # type: ignore[import-untyped]
+
+from django.contrib import admin
+from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
+from django.db import models
+from django.utils import timezone
+from django.db.utils import OperationalError
+from django.contrib.auth import get_user_model
+from django.urls import reverse_lazy
+
+from django_stubs_ext.db.models import TypedModelMeta
+
+from .abid import (
+    ABID,
+    ABID_LEN,
+    ABID_RAND_LEN,
+    ABID_SUFFIX_LEN,
+    DEFAULT_ABID_PREFIX,
+    DEFAULT_ABID_URI_SALT,
+    abid_part_from_prefix,
+    abid_hashes_from_values,
+    ts_from_abid,
+    abid_part_from_ts,
+)
+
+####################################################
+
+
+# Database Field for typeid/ulid style IDs with a prefix, e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ
+ABIDField = partial(
+    CharIDField,
+    max_length=ABID_LEN,
+    help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
+    default=None,
+    null=True,
+    blank=True,
+    db_index=True,
+    unique=True,
+)
+
+def get_or_create_system_user_pk(username='system'):
+    """Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
+
+    User = get_user_model()
+
+    # if only one user exists total, return that user
+    if User.objects.filter(is_superuser=True).count() == 1:
+        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
+
+    # otherwise, create a dedicated "system" user
+    user, _was_created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
+    return user.pk
+
+
+class AutoDateTimeField(models.DateTimeField):
+    # def pre_save(self, model_instance, add):
+    #     return timezone.now()
+    pass
+
+class ABIDError(Exception):
+    pass
+
+
+class ABIDModel(models.Model):
+    """
+    Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
+    """
+    abid_prefix: str = DEFAULT_ABID_PREFIX            # e.g. 'tag_'
+    abid_ts_src = 'self.created_at'                  # e.g. 'self.created_at'
+    abid_uri_src = 'None'                            # e.g. 'self.uri'                (MUST BE SET)
+    abid_subtype_src = 'self.__class__.__name__'     # e.g. 'self.extractor'
+    abid_rand_src = 'self.id'                        # e.g. 'self.uuid' or 'self.id'
+    abid_salt: str = DEFAULT_ABID_URI_SALT           # combined with self.uri to anonymize hashes on a per-install basis (default is shared globally with all users, means everyone will hash ABC to -> 123 the same around the world, makes it easy to share ABIDs across installs and see if they are for the same URI. Change this if you dont want your hashes to be guessable / in the same hash space as all other users)
+    abid_drift_allowed: bool = False                 # set to True to allow abid_field values to change after a fixed ABID has been issued (NOT RECOMMENDED: means values can drift out of sync from original ABID)
+
+    # id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+    # abid = ABIDField(prefix=abid_prefix)
+
+    # created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
+    # created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+    # modified_at = models.DateTimeField(auto_now=True)
+
+    _prefetched_objects_cache: Dict[str, Any]
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    @admin.display(description='Summary')
+    def __str__(self) -> str:
+        return f'[{self.abid or (self.abid_prefix + "NEW")}] {self.__class__.__name__} {eval(self.abid_uri_src)}'
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Overriden __init__ method ensures we have a stable creation timestamp that fields can use within initialization code pre-saving to DB."""
+        super().__init__(*args, **kwargs)
+        # pre-compute a stable timestamp of the obj init time (with abid.ts precision limit applied) for use when object is first created,
+        # some other fields depend on a timestamp at creation time, and it's nice to have one common timestamp they can all share.
+        # Used as an alternative to auto_now_add=True + auto_now=True which can produce two different times & requires saving to DB to get the TS.
+        # (ordinarily fields cant depend on other fields until the obj is saved to db and recalled)
+        self._init_timestamp = ts_from_abid(abid_part_from_ts(timezone.now()))
+
+    def clean(self, abid_drift_allowed: bool | None=None) -> None:
+        if self._state.adding:
+            # only runs once when a new object is first saved to the DB
+            # sets self.id, self.pk, self.created_by, self.created_at, self.modified_at
+            self._previous_abid = None
+            self.abid = str(self.issue_new_abid())
+
+        else:
+            # otherwise if updating, make sure none of the field changes would invalidate existing ABID
+            abid_diffs = self.ABID_FRESH_DIFFS
+            if abid_diffs:
+                # change has invalidated the existing ABID, raise a nice ValidationError pointing out which fields caused the issue
+
+                keys_changed = ', '.join(diff['abid_src'] for diff in abid_diffs.values())
+                full_summary = (
+                    f"This {self.__class__.__name__}(abid={str(self.ABID)}) was assigned a fixed, unique ID (ABID) based on its contents when it was created. " +
+                    f"\nYou must reduce your changes to not affect these fields [{keys_changed}], or create a new {self.__class__.__name__} object instead."
+                )
+
+                change_error = ValidationError({
+                    **{
+                        # url: ValidationError('Cannot update self.url= https://example.com/old -> https://example.com/new ...')
+                        diff['abid_src'].replace('self.', '')
+                            if (diff['old_val'] != diff['new_val']) and hasattr(self, diff['abid_src'].replace('self.', ''))
+                            else NON_FIELD_ERRORS
+                        : ValidationError(
+                            'Cannot update %(abid_src)s= "%(old_val)s" -> "%(new_val)s" (would alter %(model)s.ABID.%(key)s=%(old_hash)s to %(new_hash)s)',
+                            code='ABIDConflict',
+                            params=diff,
+                        )
+                        for diff in abid_diffs.values()
+                    },
+                    NON_FIELD_ERRORS: ValidationError(full_summary),
+                })
+
+                allowed_to_invalidate_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed
+                if allowed_to_invalidate_abid:
+                    # print(f'\n#### WARNING: Change allowed despite it invalidating the ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})!', self.abid)
+                    # print(change_error)
+                    # print('--------------------------------------------------------------------------------------------------')
+                    pass
+                else:
+                    print(f'\n#### ERROR:   Change blocked because it would invalidate ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})', self.abid)
+                    print(change_error)
+                    print('--------------------------------------------------------------------------------------------------')
+                    raise change_error
+
+    def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None:
+        """Overriden save method ensures new ABID is generated while a new object is first saving."""
+
+        self.clean(abid_drift_allowed=abid_drift_allowed)
+
+        return super().save(*args, **kwargs)
+    
+    @classmethod
+    def id_from_abid(cls, abid: str) -> str:
+        return str(cls.objects.only('pk').get(abid=cls.abid_prefix + str(abid).split('_', 1)[-1]).pk)
+
+    @property
+    def ABID_SOURCES(self) -> Dict[str, str]:
+        """"Get the dict of fresh ABID component values based on the live object's properties."""
+        assert self.abid_prefix
+        return {
+            'prefix': 'self.abid_prefix',             # defined as static class vars at build time
+            'ts': self.abid_ts_src,
+            'uri': self.abid_uri_src,
+            'subtype': self.abid_subtype_src,
+            'rand': self.abid_rand_src,
+            'salt': 'self.abid_salt',                 # defined as static class vars at build time
+        }
+
+    @property
+    def ABID_FRESH_VALUES(self) -> Dict[str, Any]:
+        """"Get the dict of fresh ABID component values based on the live object's properties."""
+        abid_sources = self.ABID_SOURCES
+        assert all(src != 'None' for src in abid_sources.values())
+        return {
+            'prefix': eval(abid_sources['prefix']),
+            'ts': eval(abid_sources['ts']),
+            'uri': eval(abid_sources['uri']),
+            'subtype': eval(abid_sources['subtype']),
+            'rand': eval(abid_sources['rand']),
+            'salt': eval(abid_sources['salt']),
+        }
+    
+    @property
+    def ABID_FRESH_HASHES(self) -> Dict[str, str]:
+        """"Get the dict of fresh ABID component hashes based on the live object's properties."""
+        abid_values = self.ABID_FRESH_VALUES
+        assert all(val for val in abid_values.values())
+        return abid_hashes_from_values(
+            prefix=abid_values['prefix'],
+            ts=abid_values['ts'],
+            uri=abid_values['uri'],
+            subtype=abid_values['subtype'],
+            rand=abid_values['rand'],
+            salt=abid_values['salt'],
+        )
+    
+    @property
+    def ABID_FRESH_DIFFS(self) -> Dict[str, Dict[str, Any]]:
+        """Get the dict of discrepancies between the existing saved ABID and a new fresh ABID computed based on the live object."""
+        existing_abid = self.ABID
+        existing_values = {} if self._state.adding else self.__class__.objects.get(pk=self.pk).ABID_FRESH_VALUES
+        abid_sources = self.ABID_SOURCES
+        fresh_values = self.ABID_FRESH_VALUES
+        fresh_hashes = self.ABID_FRESH_HASHES
+        return {
+            key: {
+                'key': key,
+                'model': self.__class__.__name__,
+                'pk': self.pk,
+                'abid_src': abid_sources[key],
+                'old_val': existing_values.get(key, None),
+                'old_hash': getattr(existing_abid, key),
+                'new_val': fresh_values[key],
+                'new_hash': new_hash,
+                'summary': f'{abid_sources[key]}= "{existing_values.get(key, None)}" -> "{fresh_values[key]}" (would alter {self.__class__.__name__.lower()}.ABID.{key}={getattr(existing_abid, key)} to {new_hash})',
+            }
+            for key, new_hash in fresh_hashes.items()
+            if getattr(existing_abid, key) != new_hash
+        }
+
+    def issue_new_abid(self, overwrite=False) -> ABID:
+        """
+        Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB).
+        """
+        if not overwrite:
+            assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
+        assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'
+
+        # Setup Field defaults to be ready for ABID generation
+        self.abid = None
+        self.id = self.id or uuid4()
+        self.pk = self.id
+        self.created_at = self.created_at or self._init_timestamp  # cut off precision to match precision of TS component
+        self.modified_at = self.modified_at or self.created_at
+        self.created_by_id = (hasattr(self, 'created_by_id') and self.created_by_id) or get_or_create_system_user_pk()
+        
+        # Compute fresh ABID values & hashes based on object's live properties
+        abid_fresh_values = self.ABID_FRESH_VALUES
+        assert all(abid_fresh_values.values()), f'All ABID_FRESH_VALUES must be set {abid_fresh_values}'
+        abid_fresh_hashes = self.ABID_FRESH_HASHES
+        assert all(abid_fresh_hashes.values()), f'All ABID_FRESH_HASHES must be able to be generated {abid_fresh_hashes}'
+        
+        new_abid = ABID(**abid_fresh_hashes)
+        
+        assert new_abid.ulid and new_abid.uuid and new_abid.typeid, f'Failed to calculate {abid_fresh_values["prefix"]}_ABID for {self.__class__.__name__}'
+
+        return new_abid
+
+    @property
+    def ABID(self) -> ABID:
+        """
+        Get the object's existing ABID (from self.abid if it's already saved to DB, otherwise generated fresh)
+        e.g. -> ABID(ts='01HX9FPYTR', uri='E4A5CCD9', subtype='00', rand='ZYEBQE')
+        """
+
+        if self.abid:
+            return ABID.parse(cast(str, self.abid))
+        
+        return self.issue_new_abid()
+
+    # These are all example helpers to make it easy to access alternate formats of the ABID.*, only add them if you actually need them
+    # @property
+    # def UUID(self) -> UUID:
+    #     """
+    #     Get a uuid.UUID (v4) representation of the object's ABID.
+    #     """
+    #     return self.ABID.uuid
+    
+    # @property
+    # def uuid(self) -> str:
+    #     """
+    #     Get a str uuid.UUID (v4) representation of the object's ABID.
+    #     """
+    #     return str(self.ABID.uuid)
+    
+    # @property
+    # def ULID(self) -> ULID:
+    #     """
+    #     Get a ulid.ULID representation of the object's ABID.
+    #     """
+    #     return self.ABID.ulid
+
+    # @property
+    # def TypeID(self) -> TypeID:
+    #     """
+    #     Get a typeid.TypeID (stripe-style) representation of the object's ABID.
+    #     """
+    #     return self.ABID.typeid
+    
+    @property
+    def api_url(self) -> str:
+        """
+        Compute the REST API URL to access this object.
+        e.g. /api/v1/core/snapshot/snp_01BJQMF54D093DXEAWZ6JYRP
+        """
+        return reverse_lazy('api-1:get_any', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
+
+    @property
+    def api_docs_url(self) -> str:
+        """
+        Compute the REST API Documentation URL to learn about accessing this object.
+        e.g. /api/v1/docs#/Core%20Models/api_v1_core_get_snapshots
+        """
+        return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}'
+
+    @property
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
+
+    def get_absolute_url(self):
+        return self.api_docs_url
+
+
+
+class ModelWithHealthStats(models.Model):
+    num_uses_failed = models.PositiveIntegerField(default=0)
+    num_uses_succeeded = models.PositiveIntegerField(default=0)
+    
+    class Meta:
+        abstract = True
+    
+    def record_health_failure(self) -> None:
+        self.num_uses_failed += 1
+        self.save()
+
+    def record_health_success(self) -> None:
+        self.num_uses_succeeded += 1
+        self.save()
+        
+    def reset_health(self) -> None:
+        # move all the failures to successes when resetting so we dont lose track of the total count
+        self.num_uses_succeeded = self.num_uses_failed + self.num_uses_succeeded
+        self.num_uses_failed = 0
+        self.save()
+        
+    @property
+    def health(self) -> int:
+        total_uses = max((self.num_uses_failed + self.num_uses_succeeded, 1))
+        success_pct = (self.num_uses_succeeded / total_uses) * 100
+        return round(success_pct)
+
+
+
+
+
+
+
+
+
+
+####################################################
+
+# Django helpers
+def find_all_abid_prefixes() -> Dict[str, type[models.Model]]:
+    """
+    Return the mapping of all ABID prefixes to their models.
+    e.g. {'tag_': core.models.Tag, 'snp_': core.models.Snapshot, ...}
+    """
+    import django.apps
+    prefix_map = {}
+
+    for model in django.apps.apps.get_models():
+        abid_prefix = getattr(model, 'abid_prefix', None)
+        if abid_prefix:
+            prefix_map[abid_prefix] = model
+    return prefix_map
+
+def find_prefix_for_abid(abid: ABID) -> str:
+    """
+    Find the correct prefix for a given ABID that may have be missing a prefix (slow).
+    e.g. ABID('obj_01BJQMF54D093DXEAWZ6JYRPAQ') -> 'snp_'
+    """
+    # if existing abid prefix is correct, lookup is easy
+    model = find_model_from_abid(abid)
+    if model:
+        assert issubclass(model, ABIDModel)
+        return model.abid_prefix
+
+    # prefix might be obj_ or missing, fuzzy-search to find any object that matches
+    return find_obj_from_abid_rand(abid)[0].abid_prefix
+
+def find_model_from_abid_prefix(prefix: str) -> type[ABIDModel] | None:
+    """
+    Return the Django Model that corresponds to a given ABID prefix.
+    e.g. 'tag_' -> core.models.Tag
+    """
+    prefix = abid_part_from_prefix(prefix)
+
+    import django.apps
+
+    for model in django.apps.apps.get_models():
+        if not issubclass(model, ABIDModel): continue   # skip non-ABID-enabled models
+        if not hasattr(model, 'objects'): continue      # skip abstract models
+
+        if (model.abid_prefix == prefix):
+            return model
+
+    return None
+
+def find_model_from_abid(abid: ABID) -> type[models.Model] | None:
+    """
+    Shortcut for find_model_from_abid_prefix(abid.prefix)
+    """
+    return find_model_from_abid_prefix(abid.prefix)
+
+def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDModel]:
+    """
+    Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
+    e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
+    Honestly should only be used for debugging, no reason to expose this ability to users.
+    """
+
+    # convert str to ABID if necessary
+    if isinstance(rand, ABID):
+        abid: ABID = rand
+    else:
+        rand = str(rand)
+        if len(rand) < ABID_SUFFIX_LEN:
+            padding_needed = ABID_SUFFIX_LEN - len(rand)
+            rand = ('0'*padding_needed) + rand
+        abid = ABID.parse(rand)
+
+    import django.apps
+
+    partial_matches: List[ABIDModel] = []
+
+    models_to_try = cast(Set[type[models.Model]], set(filter(bool, (
+        model,
+        find_model_from_abid(abid),
+        *django.apps.apps.get_models(),
+    ))))
+    # print(abid, abid.rand, abid.uuid, models_to_try)
+
+    for model in models_to_try:
+        if not issubclass(model, ABIDModel): continue   # skip Models that arent ABID-enabled
+        if not hasattr(model, 'objects'): continue      # skip abstract Models
+        assert hasattr(model, 'objects')                # force-fix for type hint nit about missing manager https://github.com/typeddjango/django-stubs/issues/1684
+
+        # continue on to try fuzzy searching by randomness portion derived from uuid field
+        try:
+            qs = []
+            if hasattr(model, 'abid'):
+                qs = model.objects.filter(abid__endswith=abid.rand)
+            elif hasattr(model, 'uuid'):
+                qs = model.objects.filter(uuid__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
+            elif hasattr(model, 'id'):
+                # NOTE: this only works on SQLite where every column is a string
+                # other DB backends like postgres dont let you do __endswith if this is a BigAutoInteger field
+                
+                # try to search for uuid=...-2354352
+                # try to search for id=...2354352
+                # try to search for id=2354352
+                qs = model.objects.filter(
+                    models.Q(id__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
+                    | models.Q(id__endswith=abid.rand)
+                    | models.Q(id__startswith=str(int(abid.rand)) if abid.rand.isdigit() else abid.rand)
+                )
+
+            for obj in qs:
+                if abid in (str(obj.ABID), str(obj.id), str(obj.pk), str(obj.abid)):
+                    # found exact match, no need to keep iterating
+                    return [obj]
+                partial_matches.append(obj)
+        except OperationalError as err:
+            print(f'[!] WARNING: Got error while trying to iterate through QuerySet for {model}:', err, '\n')
+
+    return partial_matches
+
+def find_obj_from_abid(abid: ABID, model=None, fuzzy=False) -> Any:
+    """
+    Find an object with a given ABID by filtering possible models for a matching abid/uuid/id (fast).
+    e.g. 'snp_01BJQMF54D093DXEAWZ6JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
+    """
+
+    model = model or find_model_from_abid(abid)
+    assert model, f'Could not find model that could match this ABID type: {abid}'
+
+    try:
+        if hasattr(model, 'abid'):
+            return model.objects.get(abid__endswith=abid.suffix)
+        if hasattr(model, 'uuid'):
+            return model.objects.get(uuid=abid.uuid)
+        return model.objects.get(id=abid.uuid)
+    except model.DoesNotExist:
+        # if the model has an abid field then it shouldve matched, pointless to fuzzy search in that case
+        if hasattr(model, 'abid') or (not fuzzy):
+            raise
+
+    # continue on to try fuzzy searching by randomness portion derived from uuid field
+    match_by_rand = find_obj_from_abid_rand(abid, model=model)
+    if match_by_rand:
+        if match_by_rand[0].abid_prefix != abid.prefix:
+            print(f'[!] WARNING: fetched object {match_by_rand} even though prefix {abid.prefix} doesnt match!', abid, '\n')
+        return match_by_rand
+
+    raise model.DoesNotExist
+
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
@@ -0,0 +1,571 @@
+__package__ = 'archivebox.actors'
+
+import os
+import time
+from typing import ClassVar, Generic, TypeVar, Any, Literal, Type, Iterable, cast, get_args
+from datetime import timedelta
+from multiprocessing import Process, cpu_count
+from threading import Thread, get_native_id
+
+import psutil
+from rich import print
+from statemachine import State, StateMachine, registry
+from statemachine.mixins import MachineMixin
+
+from django import db
+from django.db.models import QuerySet, sql, Q
+from django.db.models import Model as DjangoModel
+from django.utils import timezone
+from django.utils.functional import classproperty
+
+from .models import ModelWithStateMachine
+
+# from archivebox.logging_util import TimedProgress
+
+class ActorObjectAlreadyClaimed(Exception):
+    """Raised when the Actor tries to claim the next object from the queue but it's already been claimed by another Actor"""
+    pass
+
+class ActorQueueIsEmpty(Exception):
+    """Raised when the Actor tries to get the next object from the queue but it's empty"""
+    pass
+
+CPU_COUNT = cpu_count()
+DEFAULT_MAX_TICK_TIME = 60
+DEFAULT_MAX_CONCURRENT_ACTORS = min(max(2, int(CPU_COUNT * 0.6)), 8)   # 2 < 60% * num available cpu cores < 8
+
+limit = lambda n, max: min(n, max)
+
+LaunchKwargs = dict[str, Any]
+ObjectState = State | str
+ObjectStateList = Iterable[ObjectState]
+
+ModelType = TypeVar('ModelType', bound=ModelWithStateMachine)
+
+class ActorType(Generic[ModelType]):
+    """
+    Base class for all actors. Usage:
+    
+    class FaviconActor(ActorType[FaviconArchiveResult]):
+        FINAL_STATES: ClassVar[tuple[str, ...]] = ('succeeded', 'failed', 'skipped')
+        ACTIVE_STATE: ClassVar[str] = 'started'
+        
+        @classmethod
+        def qs(cls) -> QuerySet[FaviconArchiveResult]:
+            return ArchiveResult.objects.filter(extractor='favicon')   # or leave the default: FaviconArchiveResult.objects.all()
+    """
+    
+    ### Class attributes (defined on the class at compile-time when ActorType[MyModel] is defined)
+    Model: Type[ModelType]
+    StateMachineClass: Type[StateMachine]
+    
+    STATE_FIELD_NAME: ClassVar[str]
+    ACTIVE_STATE: ClassVar[ObjectState]
+    FINAL_STATES: ClassVar[ObjectStateList]
+    EVENT_NAME: ClassVar[str] = 'tick'                                    # the event name to trigger on the obj.sm: StateMachine (usually 'tick')
+    
+    CLAIM_ORDER: ClassVar[tuple[str, ...]] = ('retry_at',)                # the .order(*args) to claim the queue objects in, use ('?',) for random order
+    CLAIM_FROM_TOP_N: ClassVar[int] = CPU_COUNT * 10                      # the number of objects to consider when atomically getting the next object from the queue
+    CLAIM_ATOMIC: ClassVar[bool] = True                                   # whether to atomically fetch+claim the next object in one query, or fetch and lock it in two queries
+    
+    MAX_TICK_TIME: ClassVar[int] = DEFAULT_MAX_TICK_TIME                  # maximum duration in seconds to process a single object
+    MAX_CONCURRENT_ACTORS: ClassVar[int] = DEFAULT_MAX_CONCURRENT_ACTORS  # maximum number of concurrent actors that can be running at once
+    
+    _SPAWNED_ACTOR_PIDS: ClassVar[list[psutil.Process]] = []      # used to record all the pids of Actors spawned on the class
+    
+    ### Instance attributes (only used within an actor instance inside a spawned actor thread/process)
+    pid: int
+    idle_count: int = 0
+    launch_kwargs: LaunchKwargs = {}
+    mode: Literal['thread', 'process'] = 'process'
+    
+    def __init_subclass__(cls) -> None:
+        """
+        Executed at class definition time (i.e. during import of any file containing class MyActor(ActorType[MyModel]): ...).
+        Loads the django Model from the Generic[ModelType] TypeVar arg and populates any missing class-level config using it.
+        """
+        if getattr(cls, 'Model', None) is None:
+            cls.Model = cls._get_model_from_generic_typevar()
+        cls._populate_missing_classvars_from_model(cls.Model)
+    
+    def __init__(self, mode: Literal['thread', 'process']|None=None, **launch_kwargs: LaunchKwargs):
+        """
+        Executed right before the Actor is spawned to create a unique Actor instance for that thread/process.
+        actor_instance.runloop() is then executed from inside the newly spawned thread/process.
+        """
+        self.mode = mode or self.mode
+        self.launch_kwargs = launch_kwargs or dict(self.launch_kwargs)
+    
+
+    ### Private Helper Methods: Not desiged to be overridden by subclasses or called by anything outside of this class
+    
+    @classproperty
+    def name(cls) -> str:
+        return cls.__name__  # type: ignore
+    
+    def __str__(self) -> str:
+        return repr(self)
+    
+    def __repr__(self) -> str:
+        """-> FaviconActor[pid=1234]"""
+        label = 'pid' if self.mode == 'process' else 'tid'
+        return f'[underline]{self.name}[/underline]\\[{label}={self.pid}]'
+    
+    @staticmethod
+    def _state_to_str(state: ObjectState) -> str:
+        """Convert a statemachine.State, models.TextChoices.choices value, or Enum value to a str"""
+        return str(state.value) if isinstance(state, State) else str(state)
+    
+    @staticmethod
+    def _sql_for_select_top_n_candidates(qs: QuerySet, claim_from_top_n: int=CLAIM_FROM_TOP_N) -> tuple[str, tuple[Any, ...]]:
+        """Get the SQL for selecting the top N candidates from the queue (to claim one from)"""
+        queryset = qs.only('id')[:claim_from_top_n]
+        select_sql, select_params = compile_sql_select(queryset)
+        return select_sql, select_params
+    
+    @staticmethod
+    def _sql_for_update_claimed_obj(qs: QuerySet, update_kwargs: dict[str, Any]) -> tuple[str, tuple[Any, ...]]:
+        """Get the SQL for updating a claimed object to mark it as ACTIVE"""
+        # qs.update(status='started', retry_at=<now + MAX_TICK_TIME>)
+        update_sql, update_params = compile_sql_update(qs, update_kwargs=update_kwargs)
+        # e.g. UPDATE core_archiveresult SET status='%s', retry_at='%s' WHERE status NOT IN ('succeeded', 'failed', 'sealed', 'started') AND retry_at <= '2024-11-04 10:14:33.240903'
+        return update_sql, update_params
+    
+    @classmethod
+    def _get_model_from_generic_typevar(cls) -> Type[ModelType]:
+        """Get the django Model from the Generic[ModelType] TypeVar arg (and check that it inherits from django.db.models.Model)"""
+        # cls.__orig_bases__ is non-standard and may be removed in the future! if this breaks,
+        # we can just require the inerited class to define the Model as a classvar manually, e.g.:
+        #     class SnapshotActor(ActorType[Snapshot]):
+        #         Model: ClassVar[Type[Snapshot]] = Snapshot
+        # https://stackoverflow.com/questions/57706180/generict-base-class-how-to-get-type-of-t-from-within-instance
+        Model = get_args(cls.__orig_bases__[0])[0]   # type: ignore
+        assert issubclass(Model, DjangoModel), f'{cls.__name__}.Model must be a valid django Model'
+        return cast(Type[ModelType], Model)
+    
+    @staticmethod
+    def _get_state_machine_cls(Model: Type[ModelType]) -> Type[StateMachine]:
+        """Get the StateMachine class for the given django Model that inherits from MachineMixin"""
+        assert issubclass(Model, MachineMixin), f'{Model.__name__} must inherit from MachineMixin and define a .state_machine_name: str'
+        model_state_machine_name = getattr(Model, 'state_machine_name', None)
+        if model_state_machine_name:
+            StateMachine = registry.get_machine_cls(model_state_machine_name)
+            assert issubclass(StateMachine, StateMachine)
+            return StateMachine
+        raise NotImplementedError(f'ActorType[{Model.__name__}] must define .state_machine_name: str that points to a valid StateMachine')
+    
+    @classmethod
+    def _get_state_machine_instance(cls, obj: ModelType) -> StateMachine:
+        """Get the StateMachine instance for the given django Model instance (and check that it is a valid instance of cls.StateMachineClass)"""
+        obj_statemachine = None
+        state_machine_attr = getattr(obj, 'state_machine_attr', 'sm')
+        try:
+            obj_statemachine = getattr(obj, state_machine_attr)
+        except Exception:
+            pass
+        
+        if not isinstance(obj_statemachine, cls.StateMachineClass):
+            raise Exception(f'{cls.__name__}: Failed to find a valid StateMachine instance at {type(obj).__name__}.{state_machine_attr}')
+            
+        return obj_statemachine
+    
+    @classmethod
+    def _populate_missing_classvars_from_model(cls, Model: Type[ModelType]):
+        """Check that the class variables are set correctly based on the ModelType"""
+        
+        # check that Model is the same as the Generic[ModelType] parameter in the class definition
+        cls.Model = getattr(cls, 'Model', None) or Model
+        if cls.Model != Model:
+            raise ValueError(f'{cls.__name__}.Model must be set to the same Model as the Generic[ModelType] parameter in the class definition')
+        
+        # check that Model has a valid StateMachine with the required event defined on it
+        cls.StateMachineClass = getattr(cls, 'StateMachineClass', None) or cls._get_state_machine_cls(cls.Model)
+        assert isinstance(cls.EVENT_NAME, str), f'{cls.__name__}.EVENT_NAME must be a str, got: {type(cls.EVENT_NAME).__name__} instead'
+        assert hasattr(cls.StateMachineClass, cls.EVENT_NAME), f'StateMachine {cls.StateMachineClass.__name__} must define a {cls.EVENT_NAME} event ({cls.__name__}.EVENT_NAME = {cls.EVENT_NAME})'
+        
+        # check that Model uses .id as its primary key field
+        primary_key_field = cls.Model._meta.pk.name
+        if primary_key_field != 'id':
+            raise NotImplementedError(f'Actors currently only support models that use .id as their primary key field ({cls.__name__} uses {cls.__name__}.{primary_key_field} as primary key)')
+        
+        # check for STATE_FIELD_NAME classvar or set it from the model's state_field_name attr
+        if not getattr(cls, 'STATE_FIELD_NAME', None):
+            if hasattr(cls.Model, 'state_field_name'):
+                cls.STATE_FIELD_NAME = getattr(cls.Model, 'state_field_name')
+            else:
+                raise NotImplementedError(f'{cls.__name__} must define a STATE_FIELD_NAME: ClassVar[str] (e.g. "status") or have a .state_field_name attr on its Model')
+        assert isinstance(cls.STATE_FIELD_NAME, str), f'{cls.__name__}.STATE_FIELD_NAME must be a str, got: {type(cls.STATE_FIELD_NAME).__name__} instead'
+            
+        # check for FINAL_STATES classvar or set it from the model's final_states attr
+        if not getattr(cls, 'FINAL_STATES', None):
+            cls.FINAL_STATES = cls.StateMachineClass.final_states
+            if not cls.FINAL_STATES:
+                raise NotImplementedError(f'{cls.__name__} must define a non-empty FINAL_STATES: ClassVar[list[str]] (e.g. ["sealed"]) or have a {cls.Model.__name__}.state_machine_name pointing to a StateMachine that provides .final_states')
+        cls.FINAL_STATES = [cls._state_to_str(state) for state in cls.FINAL_STATES]
+        assert all(isinstance(state, str) for state in cls.FINAL_STATES), f'{cls.__name__}.FINAL_STATES must be a list[str], got: {type(cls.FINAL_STATES).__name__} instead'
+        
+        # check for ACTIVE_STATE classvar or set it from the model's active_state attr
+        if not getattr(cls, 'ACTIVE_STATE', None):
+            raise NotImplementedError(f'{cls.__name__} must define an ACTIVE_STATE: ClassVar[State] (e.g. SnapshotMachine.started) ({cls.Model.__name__}.{cls.STATE_FIELD_NAME} gets set to this value to mark objects as actively processing)')
+        assert isinstance(cls.ACTIVE_STATE, (State, str)), f'{cls.__name__}.ACTIVE_STATE must be a statemachine.State | str, got: {type(cls.ACTIVE_STATE).__name__} instead'
+        
+        # check the other ClassVar attributes for valid values
+        assert cls.CLAIM_ORDER and isinstance(cls.CLAIM_ORDER, tuple) and all(isinstance(order, str) for order in cls.CLAIM_ORDER), f'{cls.__name__}.CLAIM_ORDER must be a non-empty tuple[str, ...], got: {type(cls.CLAIM_ORDER).__name__} instead'
+        assert cls.CLAIM_FROM_TOP_N > 0, f'{cls.__name__}.CLAIM_FROM_TOP_N must be a positive int, got: {cls.CLAIM_FROM_TOP_N} instead'
+        assert cls.MAX_TICK_TIME >= 1, f'{cls.__name__}.MAX_TICK_TIME must be a positive int > 1, got: {cls.MAX_TICK_TIME} instead'
+        assert cls.MAX_CONCURRENT_ACTORS >= 1, f'{cls.__name__}.MAX_CONCURRENT_ACTORS must be a positive int >=1, got: {cls.MAX_CONCURRENT_ACTORS} instead'
+        assert isinstance(cls.CLAIM_ATOMIC, bool), f'{cls.__name__}.CLAIM_ATOMIC must be a bool, got: {cls.CLAIM_ATOMIC} instead'
+
+    @classmethod
+    def _fork_actor_as_thread(cls, **launch_kwargs: LaunchKwargs) -> int:
+        """Spawn a new background thread running the actor's runloop"""
+        actor = cls(mode='thread', **launch_kwargs)
+        bg_actor_thread = Thread(target=actor.runloop)
+        bg_actor_thread.start()
+        assert bg_actor_thread.native_id is not None
+        return bg_actor_thread.native_id
+    
+    @classmethod
+    def _fork_actor_as_process(cls, **launch_kwargs: LaunchKwargs) -> int:
+        """Spawn a new background process running the actor's runloop"""
+        actor = cls(mode='process', **launch_kwargs)
+        bg_actor_process = Process(target=actor.runloop)
+        bg_actor_process.start()
+        assert bg_actor_process.pid is not None
+        cls._SPAWNED_ACTOR_PIDS.append(psutil.Process(pid=bg_actor_process.pid))
+        return bg_actor_process.pid
+    
+    
+    ### Class Methods: Called by Orchestrator on ActorType class before it has been spawned
+    
+    @classmethod
+    def get_running_actors(cls) -> list[int]:
+        """returns a list of pids of all running actors of this type"""
+        # WARNING: only works for process actors, not thread actors
+        if cls.mode == 'thread':
+            raise NotImplementedError('get_running_actors() is not implemented for thread actors')
+        return [
+            proc.pid for proc in cls._SPAWNED_ACTOR_PIDS
+            if proc.is_running() and proc.status() != 'zombie'
+        ]
+        
+    @classmethod
+    def get_actors_to_spawn(cls, queue: QuerySet, running_actors: list[int]) -> list[LaunchKwargs]:
+        """Get a list of launch kwargs for the number of actors to spawn based on the queue and currently running actors"""
+        queue_length = queue.count()
+        if not queue_length:                                      # queue is empty, spawn 0 actors
+            return []
+        
+        # WARNING:
+        # spawning new actors processes is slow/expensive, avoid spawning many actors at once in a single orchestrator tick.
+        # limit to spawning 1 or 2 at a time per orchestrator tick, and let the next tick handle starting another couple.
+        # DONT DO THIS:
+        # if queue_length > 20:                      # queue is extremely long, spawn maximum actors at once!
+        #   num_to_spawn_this_tick = cls.MAX_CONCURRENT_ACTORS
+        
+        if queue_length > 10:    
+            num_to_spawn_this_tick = 2  # spawn more actors per tick if queue is long
+        else:
+            num_to_spawn_this_tick = 1  # spawn fewer actors per tick if queue is short
+        
+        num_remaining = cls.MAX_CONCURRENT_ACTORS - len(running_actors)
+        num_to_spawn_now: int = limit(num_to_spawn_this_tick, num_remaining)
+        
+        actors_launch_kwargs: list[LaunchKwargs] = num_to_spawn_now * [{**cls.launch_kwargs}]
+        return actors_launch_kwargs
+        
+    @classmethod
+    def start(cls, mode: Literal['thread', 'process']='process', **launch_kwargs: LaunchKwargs) -> int:
+        if mode == 'thread':
+            return cls._fork_actor_as_thread(**launch_kwargs)
+        elif mode == 'process':
+            return cls._fork_actor_as_process(**launch_kwargs)
+        raise ValueError(f'Invalid actor mode: {mode} must be "thread" or "process"')
+    
+    @classproperty
+    def qs(cls) -> QuerySet[ModelType]:
+        """
+        Get the unfiltered and unsorted QuerySet of all objects that this Actor might care about.
+        Override this in the subclass to define the QuerySet of objects that the Actor is going to poll for new work.
+        (don't limit, order, or filter this by retry_at or status yet, Actor.get_queue() handles that part)
+        """
+        return cls.Model.objects.all()
+    
+    @classproperty
+    def final_q(cls) -> Q:
+        """Get the filter for objects that are in a final state"""
+        return Q(**{f'{cls.STATE_FIELD_NAME}__in': [cls._state_to_str(s) for s in cls.FINAL_STATES]})
+    
+    @classproperty
+    def active_q(cls) -> Q:
+        """Get the filter for objects that are actively processing right now"""
+        return Q(**{cls.STATE_FIELD_NAME: cls._state_to_str(cls.ACTIVE_STATE)})   # e.g. Q(status='started')
+    
+    @classproperty
+    def stalled_q(cls) -> Q:
+        """Get the filter for objects that are marked active but have timed out"""
+        return cls.active_q & Q(retry_at__lte=timezone.now())                     # e.g. Q(status='started') AND Q(<retry_at is in the past>)
+    
+    @classproperty
+    def future_q(cls) -> Q:
+        """Get the filter for objects that have a retry_at in the future"""
+        return Q(retry_at__gt=timezone.now())
+    
+    @classproperty
+    def pending_q(cls) -> Q:
+        """Get the filter for objects that are ready for processing."""
+        return ~(cls.active_q) & ~(cls.final_q) & ~(cls.future_q)
+    
+    @classmethod
+    def get_queue(cls, sort: bool=True) -> QuerySet[ModelType]:
+        """
+        Get the sorted and filtered QuerySet of objects that are ready for processing.
+        e.g. qs.exclude(status__in=('sealed', 'started'), retry_at__gt=timezone.now()).order_by('retry_at')
+        """
+        unsorted_qs = cls.qs.filter(cls.pending_q)
+        return unsorted_qs.order_by(*cls.CLAIM_ORDER) if sort else unsorted_qs
+
+    ### Instance Methods: Only called from within Actor instance after it has been spawned (i.e. forked as a thread or process)
+    
+    def runloop(self):
+        """The main runloop that starts running when the actor is spawned (as subprocess or thread) and exits when the queue is empty"""
+        self.on_startup()
+        obj_to_process: ModelType | None = None
+        last_error: BaseException | None = None
+        try:
+            while True:
+                # Get the next object to process from the queue
+                try:
+                    obj_to_process = cast(ModelType, self.get_next(atomic=self.atomic))
+                except (ActorQueueIsEmpty, ActorObjectAlreadyClaimed) as err:
+                    last_error = err
+                    obj_to_process = None
+                
+                # Handle the case where there is no next object to process
+                if obj_to_process:
+                    self.idle_count = 0   # reset idle count if we got an object
+                else:
+                    if self.idle_count >= 30:
+                        break             # stop looping and exit if queue is empty and we have idled for 30sec
+                    else:
+                        # print('Actor runloop()', f'pid={self.pid}', 'queue empty, rechecking...')
+                        self.idle_count += 1
+                        time.sleep(1)
+                        continue
+                
+                # Process the object by triggering its StateMachine.tick() method
+                self.on_tick_start(obj_to_process)
+                try:
+                    self.tick(obj_to_process)
+                except Exception as err:
+                    last_error = err
+                    # print(f'[red]🏃‍♂️ {self}.tick()[/red] {obj_to_process} ERROR: [red]{type(err).__name__}: {err}[/red]')
+                    db.connections.close_all()                         # always reset the db connection after an exception to clear any pending transactions
+                    self.on_tick_exception(obj_to_process, err)
+                finally:
+                    self.on_tick_end(obj_to_process)
+
+        except BaseException as err:
+            last_error = err
+            if isinstance(err, KeyboardInterrupt):
+                print()
+            else:
+                print(f'\n[red]🏃‍♂️ {self}.runloop() FATAL:[/red] {type(err).__name__}: {err}')
+                print(f'    Last processed object: {obj_to_process}')
+                raise
+        finally:
+            self.on_shutdown(last_obj=obj_to_process, last_error=last_error)
+    
+    def get_update_kwargs_to_claim_obj(self) -> dict[str, Any]:
+        """
+        Get the field values needed to mark an pending obj_to_process as being actively processing (aka claimed)
+        by the current Actor. returned kwargs will be applied using: qs.filter(id=obj_to_process.id).update(**kwargs).
+        F() expressions are allowed in field values if you need to update a field based on its current value.
+        Can be a defined as a normal method (instead of classmethod) on subclasses if it needs to access instance vars.
+        """
+        return {
+            self.STATE_FIELD_NAME: self.ACTIVE_STATE,
+            'retry_at': timezone.now() + timedelta(seconds=self.MAX_TICK_TIME),
+        }
+    
+    def get_next(self, atomic: bool | None=None) -> ModelType | None:
+        """get the next object from the queue, atomically locking it if self.atomic=True"""
+        atomic = self.CLAIM_ATOMIC if atomic is None else atomic
+        if atomic:
+            # fetch and claim the next object from in the queue in one go atomically
+            obj = self.get_next_atomic()
+        else:
+            # two-step claim: fetch the next object and lock it in a separate query
+            obj = self.get_next_non_atomic()
+        return obj
+    
+    def get_next_non_atomic(self) -> ModelType:
+        """
+        Naiively selects the top/first object from self.get_queue().order_by(*self.CLAIM_ORDER),
+        then claims it by running .update(status='started', retry_at=<now + MAX_TICK_TIME>).
+        
+        Do not use this method if there is more than one Actor racing to get objects from the same queue,
+        it will be slow/buggy as they'll compete to lock the same object at the same time (TOCTTOU race).
+        """
+        obj = self.get_queue().first()
+        if obj is None:
+            raise ActorQueueIsEmpty(f'No next object available in {self}.get_queue()')
+        
+        locked = self.get_queue().filter(id=obj.id).update(**self.get_update_kwargs_to_claim_obj())
+        if not locked:
+            raise ActorObjectAlreadyClaimed(f'Unable to lock the next {self.Model.__name__} object from {self}.get_queue().first()')
+        return obj
+        
+    def get_next_atomic(self) -> ModelType | None:
+        """
+        Selects the top n=50 objects from the queue and atomically claims a random one from that set.
+        This approach safely minimizes contention with other Actors trying to select from the same Queue.
+
+        The atomic query is roughly equivalent to the following:  (all done in one SQL query to avoid a TOCTTOU race)
+            top_candidates are selected from:   qs.order_by(*CLAIM_ORDER).only('id')[:CLAIM_FROM_TOP_N]
+            a single candidate is chosen using: qs.filter(id__in=top_n_candidates).order_by('?').first()
+            the chosen obj is claimed using:    qs.filter(id=chosen_obj).update(status=ACTIVE_STATE, retry_at=<now + MAX_TICK_TIME>)
+        """
+        # TODO: if we switch from SQLite to PostgreSQL in the future, we should change this
+        # to use SELECT FOR UPDATE instead of a subquery + ORDER BY RANDOM() LIMIT 1
+        
+        # e.g. SELECT id FROM core_archiveresult WHERE status NOT IN (...) AND retry_at <= '...' ORDER BY retry_at ASC LIMIT 50
+        qs = self.get_queue()
+        select_top_canidates_sql, select_params = self._sql_for_select_top_n_candidates(qs=qs)
+        assert select_top_canidates_sql.startswith('SELECT ')
+        
+        # e.g. UPDATE core_archiveresult SET status='%s', retry_at='%s' WHERE status NOT IN (...) AND retry_at <= '...'
+        update_claimed_obj_sql, update_params = self._sql_for_update_claimed_obj(qs=qs, update_kwargs=self.get_update_kwargs_to_claim_obj())
+        assert update_claimed_obj_sql.startswith('UPDATE ')
+        db_table = self.Model._meta.db_table  # e.g. core_archiveresult
+        
+        # subquery gets the pool of the top candidates e.g. self.get_queue().only('id')[:CLAIM_FROM_TOP_N]
+        # main query selects a random one from that pool, and claims it using .update(status=ACTIVE_STATE, retry_at=<now + MAX_TICK_TIME>)
+        # this is all done in one atomic SQL query to avoid TOCTTOU race conditions (as much as possible)
+        atomic_select_and_update_sql = f"""
+            {update_claimed_obj_sql} AND "{db_table}"."id" = (
+                SELECT "{db_table}"."id" FROM (
+                    {select_top_canidates_sql}
+                ) candidates
+                ORDER BY RANDOM()
+                LIMIT 1
+            )
+            RETURNING *;
+        """
+        try:
+            return self.Model.objects.raw(atomic_select_and_update_sql, (*update_params, *select_params))[0]
+        except KeyError:
+            if self.get_queue().exists():
+                raise ActorObjectAlreadyClaimed(f'Unable to lock the next {self.Model.__name__} object from {self}.get_queue().first()')
+            else:
+                raise ActorQueueIsEmpty(f'No next object available in {self}.get_queue()')
+
+    def tick(self, obj_to_process: ModelType) -> None:
+        """Call the object.sm.tick() method to process the object"""
+        print(f'[blue]🏃‍♂️ {self}.tick()[/blue] {obj_to_process}')
+        
+        # get the StateMachine instance from the object
+        obj_statemachine = self._get_state_machine_instance(obj_to_process)
+        
+        # trigger the event on the StateMachine instance
+        obj_tick_method = getattr(obj_statemachine, self.EVENT_NAME)  # e.g. obj_statemachine.tick()
+        obj_tick_method()
+        
+        # save the object to persist any state changes
+        obj_to_process.save()
+        
+    def on_startup(self) -> None:
+        if self.mode == 'thread':
+            self.pid = get_native_id()  # thread id
+            print(f'[green]🏃‍♂️ {self}.on_startup() STARTUP (THREAD)[/green]')
+        else:
+            self.pid = os.getpid()      # process id
+            print(f'[green]🏃‍♂️ {self}.on_startup() STARTUP (PROCESS)[/green]')
+        # abx.pm.hook.on_actor_startup(actor=self)
+        
+    def on_shutdown(self, last_obj: ModelType | None=None, last_error: BaseException | None=None) -> None:
+        if isinstance(last_error, KeyboardInterrupt) or last_error is None:
+            last_error_str = '[green](CTRL-C)[/green]'
+        elif isinstance(last_error, ActorQueueIsEmpty):
+            last_error_str = '[green](queue empty)[/green]'
+        elif isinstance(last_error, ActorObjectAlreadyClaimed):
+            last_error_str = '[green](queue race)[/green]'
+        else:
+            last_error_str = f'[red]{type(last_error).__name__}: {last_error}[/red]'
+
+        print(f'[grey53]🏃‍♂️ {self}.on_shutdown() SHUTTING DOWN[/grey53] {last_error_str}')
+        # abx.pm.hook.on_actor_shutdown(actor=self, last_obj=last_obj, last_error=last_error)
+        
+    def on_tick_start(self, obj_to_process: ModelType) -> None:
+        print(f'🏃‍♂️ {self}.on_tick_start() {obj_to_process}')
+        # abx.pm.hook.on_actor_tick_start(actor=self, obj_to_process=obj)
+        # self.timer = TimedProgress(self.MAX_TICK_TIME, prefix='      ')
+    
+    def on_tick_end(self, obj_to_process: ModelType) -> None:
+        print(f'🏃‍♂️ {self}.on_tick_end() {obj_to_process}')
+        # abx.pm.hook.on_actor_tick_end(actor=self, obj_to_process=obj_to_process)
+        # self.timer.end()
+    
+    def on_tick_exception(self, obj_to_process: ModelType, error: Exception) -> None:
+        print(f'[red]🏃‍♂️ {self}.on_tick_exception()[/red] {obj_to_process}: [red]{type(error).__name__}: {error}[/red]')
+        # abx.pm.hook.on_actor_tick_exception(actor=self, obj_to_process=obj_to_process, error=error)
+
+
+
+def compile_sql_select(queryset: QuerySet, filter_kwargs: dict[str, Any] | None=None, order_args: tuple[str, ...]=(), limit: int | None=None) -> tuple[str, tuple[Any, ...]]:
+    """
+    Compute the SELECT query SQL for a queryset.filter(**filter_kwargs).order_by(*order_args)[:limit] call
+    Returns a tuple of (sql, params) where sql is a template string containing %s (unquoted) placeholders for the params
+    
+    WARNING:
+    final_sql = sql % params  DOES NOT WORK to assemble the final SQL string because the %s placeholders are not quoted/escaped
+    they should always passed separately to the DB driver so it can do its own quoting/escaping to avoid SQL injection and syntax errors
+    """
+    assert isinstance(queryset, QuerySet), f'compile_sql_select(...) first argument must be a QuerySet, got: {type(queryset).__name__} instead'
+    assert filter_kwargs is None or isinstance(filter_kwargs, dict), f'compile_sql_select(...) filter_kwargs argument must be a dict[str, Any], got: {type(filter_kwargs).__name__} instead'
+    assert isinstance(order_args, tuple) and all(isinstance(arg, str) for arg in order_args), f'compile_sql_select(...) order_args argument must be a tuple[str, ...] got: {type(order_args).__name__} instead'
+    assert limit is None or isinstance(limit, int), f'compile_sql_select(...) limit argument must be an int, got: {type(limit).__name__} instead'
+    
+    queryset = queryset._chain()                      # type: ignore   # copy queryset to avoid modifying the original
+    if filter_kwargs:
+        queryset = queryset.filter(**filter_kwargs)
+    if order_args:
+        queryset = queryset.order_by(*order_args)
+    if limit is not None:
+        queryset = queryset[:limit]
+    query = queryset.query
+    
+    # e.g. SELECT id FROM core_archiveresult WHERE status NOT IN (%s, %s, %s) AND retry_at <= %s ORDER BY retry_at ASC LIMIT 50
+    select_sql, select_params = query.get_compiler(queryset.db).as_sql()
+    return select_sql, select_params
+
+
+def compile_sql_update(queryset: QuerySet, update_kwargs: dict[str, Any], filter_kwargs: dict[str, Any] | None=None) -> tuple[str, tuple[Any, ...]]:
+    """
+    Compute the UPDATE query SQL for a queryset.filter(**filter_kwargs).update(**update_kwargs) call
+    Returns a tuple of (sql, params) where sql is a template string containing %s (unquoted) placeholders for the params
+    
+    Based on the django.db.models.QuerySet.update() source code, but modified to return the SQL instead of executing the update
+    https://github.com/django/django/blob/611bf6c2e2a1b4ab93273980c45150c099ab146d/django/db/models/query.py#L1217
+    
+    WARNING:
+    final_sql = sql % params  DOES NOT WORK to assemble the final SQL string because the %s placeholders are not quoted/escaped
+    they should always passed separately to the DB driver so it can do its own quoting/escaping to avoid SQL injection and syntax errors
+    """
+    assert isinstance(queryset, QuerySet), f'compile_sql_update(...) first argument must be a QuerySet, got: {type(queryset).__name__} instead'
+    assert isinstance(update_kwargs, dict), f'compile_sql_update(...) update_kwargs argument must be a dict[str, Any], got: {type(update_kwargs).__name__} instead'
+    assert filter_kwargs is None or isinstance(filter_kwargs, dict), f'compile_sql_update(...) filter_kwargs argument must be a dict[str, Any], got: {type(filter_kwargs).__name__} instead'
+    
+    queryset = queryset._chain()                      # type: ignore   # copy queryset to avoid modifying the original
+    if filter_kwargs:
+        queryset = queryset.filter(**filter_kwargs)
+    queryset.query.clear_ordering(force=True)                          # clear any ORDER BY clauses
+    queryset.query.clear_limits()                                      # clear any LIMIT clauses aka slices[:n]
+    queryset._for_write = True                        # type: ignore
+    query = queryset.query.chain(sql.UpdateQuery)     # type: ignore
+    query.add_update_values(update_kwargs)            # type: ignore
+    query.annotations = {}                                             # clear any annotations
+    
+    # e.g. UPDATE core_archiveresult SET status='%s', retry_at='%s' WHERE status NOT IN (%s, %s, %s) AND retry_at <= %s
+    update_sql, update_params = query.get_compiler(queryset.db).as_sql()
+    return update_sql, update_params
@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.
@@ -1,7 +1,6 @@
 from django.apps import AppConfig


-class WorkersConfig(AppConfig):
+class ActorsConfig(AppConfig):
    default_auto_field = "django.db.models.BigAutoField"
-    name = "archivebox.workers"
-    label = "workers"
+    name = "actors"
@@ -0,0 +1,300 @@
+from typing import ClassVar, Type, Iterable
+from datetime import datetime, timedelta
+
+from statemachine.mixins import MachineMixin
+
+from django.db import models
+from django.utils import timezone
+from django.utils.functional import classproperty
+
+from statemachine import registry, StateMachine, State
+
+from django.core import checks
+
+class DefaultStatusChoices(models.TextChoices):
+    QUEUED = 'queued', 'Queued'
+    STARTED = 'started', 'Started'
+    SEALED = 'sealed', 'Sealed'
+
+
+default_status_field: models.CharField = models.CharField(choices=DefaultStatusChoices.choices, max_length=15, default=DefaultStatusChoices.QUEUED, null=False, blank=False, db_index=True)
+default_retry_at_field: models.DateTimeField = models.DateTimeField(default=timezone.now, null=False, db_index=True)
+
+ObjectState = State | str
+ObjectStateList = Iterable[ObjectState]
+
+
+class BaseModelWithStateMachine(models.Model, MachineMixin):
+    id: models.UUIDField
+    
+    StatusChoices: ClassVar[Type[models.TextChoices]]
+    
+    # status: models.CharField
+    # retry_at: models.DateTimeField
+    
+    state_machine_name: ClassVar[str]
+    state_field_name: ClassVar[str]
+    state_machine_attr: ClassVar[str] = 'sm'
+    bind_events_as_methods: ClassVar[bool] = True
+    
+    active_state: ClassVar[ObjectState]
+    retry_at_field_name: ClassVar[str]
+    
+    class Meta:
+        abstract = True
+        
+    @classmethod
+    def check(cls, sender=None, **kwargs):
+        errors = super().check(**kwargs)
+        
+        found_id_field = False
+        found_status_field = False
+        found_retry_at_field = False
+        
+        for field in cls._meta.get_fields():
+            if getattr(field, '_is_state_field', False):
+                if cls.state_field_name == field.name:
+                    found_status_field = True
+                    if getattr(field, 'choices', None) != cls.StatusChoices.choices:
+                        errors.append(checks.Error(
+                            f'{cls.__name__}.{field.name} must have choices set to {cls.__name__}.StatusChoices.choices',
+                            hint=f'{cls.__name__}.{field.name}.choices = {getattr(field, "choices", None)!r}',
+                            obj=cls,
+                            id='actors.E011',
+                        ))
+            if getattr(field, '_is_retry_at_field', False):
+                if cls.retry_at_field_name == field.name:
+                    found_retry_at_field = True
+            if field.name == 'id' and getattr(field, 'primary_key', False):
+                found_id_field = True
+                    
+        if not found_status_field:
+            errors.append(checks.Error(
+                f'{cls.__name__}.state_field_name must be defined and point to a StatusField()',
+                hint=f'{cls.__name__}.state_field_name = {cls.state_field_name!r} but {cls.__name__}.{cls.state_field_name!r} was not found or does not refer to StatusField',
+                obj=cls,
+                id='actors.E012',
+            ))
+        if not found_retry_at_field:
+            errors.append(checks.Error(
+                f'{cls.__name__}.retry_at_field_name must be defined and point to a RetryAtField()',
+                hint=f'{cls.__name__}.retry_at_field_name = {cls.retry_at_field_name!r} but {cls.__name__}.{cls.retry_at_field_name!r} was not found or does not refer to RetryAtField',
+                obj=cls,
+                id='actors.E013',
+            ))
+            
+        if not found_id_field:
+            errors.append(checks.Error(
+                f'{cls.__name__} must have an id field that is a primary key',
+                hint=f'{cls.__name__}.id = {cls.id!r}',
+                obj=cls,
+                id='actors.E014',
+            ))
+            
+        if not isinstance(cls.state_machine_name, str):
+            errors.append(checks.Error(
+                f'{cls.__name__}.state_machine_name must be a dotted-import path to a StateMachine class',
+                hint=f'{cls.__name__}.state_machine_name = {cls.state_machine_name!r}',
+                obj=cls,
+                id='actors.E015',
+            ))
+        
+        try:
+            cls.StateMachineClass
+        except Exception as err:
+            errors.append(checks.Error(
+                f'{cls.__name__}.state_machine_name must point to a valid StateMachine class, but got {type(err).__name__} {err} when trying to access {cls.__name__}.StateMachineClass',
+                hint=f'{cls.__name__}.state_machine_name = {cls.state_machine_name!r}',
+                obj=cls,
+                id='actors.E016',
+            ))
+        
+        if cls.INITIAL_STATE not in cls.StatusChoices.values:
+            errors.append(checks.Error(
+                f'{cls.__name__}.StateMachineClass.initial_state must be present within {cls.__name__}.StatusChoices',
+                hint=f'{cls.__name__}.StateMachineClass.initial_state = {cls.StateMachineClass.initial_state!r}',
+                obj=cls,
+                id='actors.E017',
+            ))
+            
+        if cls.ACTIVE_STATE not in cls.StatusChoices.values:
+            errors.append(checks.Error(
+                f'{cls.__name__}.active_state must be set to a valid State present within {cls.__name__}.StatusChoices',
+                hint=f'{cls.__name__}.active_state = {cls.active_state!r}',
+                obj=cls,
+                id='actors.E018',
+            ))
+            
+        
+        for state in cls.FINAL_STATES:
+            if state not in cls.StatusChoices.values:
+                errors.append(checks.Error(
+                    f'{cls.__name__}.StateMachineClass.final_states must all be present within {cls.__name__}.StatusChoices',
+                    hint=f'{cls.__name__}.StateMachineClass.final_states = {cls.StateMachineClass.final_states!r}',
+                    obj=cls,
+                    id='actors.E019',
+                ))
+                break
+        return errors
+    
+    @staticmethod
+    def _state_to_str(state: ObjectState) -> str:
+        """Convert a statemachine.State, models.TextChoices.choices value, or Enum value to a str"""
+        return str(state.value) if isinstance(state, State) else str(state)
+    
+    
+    @property
+    def RETRY_AT(self) -> datetime:
+        return getattr(self, self.retry_at_field_name)
+    
+    @RETRY_AT.setter
+    def RETRY_AT(self, value: datetime):
+        setattr(self, self.retry_at_field_name, value)
+        
+    @property
+    def STATE(self) -> str:
+        return getattr(self, self.state_field_name)
+    
+    @STATE.setter
+    def STATE(self, value: str):
+        setattr(self, self.state_field_name, value)
+        
+    def bump_retry_at(self, seconds: int = 10):
+        self.RETRY_AT = timezone.now() + timedelta(seconds=seconds)
+        
+    @classproperty
+    def ACTIVE_STATE(cls) -> str:
+        return cls._state_to_str(cls.active_state)
+        
+    @classproperty
+    def INITIAL_STATE(cls) -> str:
+        return cls._state_to_str(cls.StateMachineClass.initial_state)
+    
+    @classproperty
+    def FINAL_STATES(cls) -> list[str]:
+        return [cls._state_to_str(state) for state in cls.StateMachineClass.final_states]
+    
+    @classproperty
+    def FINAL_OR_ACTIVE_STATES(cls) -> list[str]:
+        return [*cls.FINAL_STATES, cls.ACTIVE_STATE]
+        
+    @classmethod
+    def extend_choices(cls, base_choices: Type[models.TextChoices]):
+        """
+        Decorator to extend the base choices with extra choices, e.g.:
+        
+        class MyModel(ModelWithStateMachine):
+        
+            @ModelWithStateMachine.extend_choices(ModelWithStateMachine.StatusChoices)
+            class StatusChoices(models.TextChoices):
+                SUCCEEDED = 'succeeded'
+                FAILED = 'failed'
+                SKIPPED = 'skipped'
+        """
+        assert issubclass(base_choices, models.TextChoices), f'@extend_choices(base_choices) must be a TextChoices class, not {base_choices.__name__}'
+        def wrapper(extra_choices: Type[models.TextChoices]) -> Type[models.TextChoices]:
+            joined = {}
+            for item in base_choices.choices:
+                joined[item[0]] = item[1]
+            for item in extra_choices.choices:
+                joined[item[0]] = item[1]
+            return models.TextChoices('StatusChoices', joined)
+        return wrapper
+        
+    @classmethod
+    def StatusField(cls, **kwargs) -> models.CharField:
+        """
+        Used on subclasses to extend/modify the status field with updated kwargs. e.g.:
+        
+        class MyModel(ModelWithStateMachine):
+            class StatusChoices(ModelWithStateMachine.StatusChoices):
+                QUEUED = 'queued', 'Queued'
+                STARTED = 'started', 'Started'
+                SEALED = 'sealed', 'Sealed'
+                BACKOFF = 'backoff', 'Backoff'
+                FAILED = 'failed', 'Failed'
+                SKIPPED = 'skipped', 'Skipped'
+        
+            status = ModelWithStateMachine.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED)
+        """
+        default_kwargs = default_status_field.deconstruct()[3]
+        updated_kwargs = {**default_kwargs, **kwargs}
+        field = models.CharField(**updated_kwargs)
+        field._is_state_field = True                    # type: ignore
+        return field
+
+    @classmethod
+    def RetryAtField(cls, **kwargs) -> models.DateTimeField:
+        """
+        Used on subclasses to extend/modify the retry_at field with updated kwargs. e.g.:
+        
+        class MyModel(ModelWithStateMachine):
+            retry_at = ModelWithStateMachine.RetryAtField(editable=False)
+        """
+        default_kwargs = default_retry_at_field.deconstruct()[3]
+        updated_kwargs = {**default_kwargs, **kwargs}
+        field = models.DateTimeField(**updated_kwargs)
+        field._is_retry_at_field = True                 # type: ignore
+        return field
+    
+    @classproperty
+    def StateMachineClass(cls) -> Type[StateMachine]:
+        """Get the StateMachine class for the given django Model that inherits from MachineMixin"""
+
+        model_state_machine_name = getattr(cls, 'state_machine_name', None)
+        if model_state_machine_name:
+            StateMachineCls = registry.get_machine_cls(model_state_machine_name)
+            assert issubclass(StateMachineCls, StateMachine)
+            return StateMachineCls
+        raise NotImplementedError(f'ActorType[{cls.__name__}] must define .state_machine_name: str that points to a valid StateMachine')
+    
+    # @classproperty
+    # def final_q(cls) -> Q:
+    #     """Get the filter for objects that are in a final state"""
+    #     return Q(**{f'{cls.state_field_name}__in': cls.final_states})
+    
+    # @classproperty
+    # def active_q(cls) -> Q:
+    #     """Get the filter for objects that are actively processing right now"""
+    #     return Q(**{cls.state_field_name: cls._state_to_str(cls.active_state)})   # e.g. Q(status='started')
+    
+    # @classproperty
+    # def stalled_q(cls) -> Q:
+    #     """Get the filter for objects that are marked active but have timed out"""
+    #     return cls.active_q & Q(retry_at__lte=timezone.now())                     # e.g. Q(status='started') AND Q(<retry_at is in the past>)
+    
+    # @classproperty
+    # def future_q(cls) -> Q:
+    #     """Get the filter for objects that have a retry_at in the future"""
+    #     return Q(retry_at__gt=timezone.now())
+    
+    # @classproperty
+    # def pending_q(cls) -> Q:
+    #     """Get the filter for objects that are ready for processing."""
+    #     return ~(cls.active_q) & ~(cls.final_q) & ~(cls.future_q)
+    
+    # @classmethod
+    # def get_queue(cls) -> QuerySet:
+    #     """
+    #     Get the sorted and filtered QuerySet of objects that are ready for processing.
+    #     e.g. qs.exclude(status__in=('sealed', 'started'), retry_at__gt=timezone.now()).order_by('retry_at')
+    #     """
+    #     return cls.objects.filter(cls.pending_q)
+
+
+class ModelWithStateMachine(BaseModelWithStateMachine):
+    StatusChoices: ClassVar[Type[DefaultStatusChoices]] = DefaultStatusChoices
+    
+    status: models.CharField = BaseModelWithStateMachine.StatusField()
+    retry_at: models.DateTimeField = BaseModelWithStateMachine.RetryAtField()
+    
+    state_machine_name: ClassVar[str]      # e.g. 'core.statemachines.ArchiveResultMachine'
+    state_field_name: ClassVar[str]        = 'status'
+    state_machine_attr: ClassVar[str]      = 'sm'
+    bind_events_as_methods: ClassVar[bool] = True
+    
+    active_state: ClassVar[str]            = StatusChoices.STARTED
+    retry_at_field_name: ClassVar[str]     = 'retry_at'
+    
+    class Meta:
+        abstract = True
@@ -0,0 +1,244 @@
+__package__ = 'archivebox.actors'
+
+import os
+import time
+import itertools
+from typing import Dict, Type, Literal, ClassVar
+from django.utils.functional import classproperty
+
+from multiprocessing import Process
+from threading import Thread, get_native_id
+
+
+from rich import print
+
+from django.db.models import QuerySet
+
+from django.apps import apps
+from .actor import ActorType
+
+class Orchestrator:
+    pid: int
+    idle_count: int = 0
+    actor_types: Dict[str, Type[ActorType]]
+    mode: Literal['thread', 'process'] = 'process'
+
+    def __init__(self, actor_types: Dict[str, Type[ActorType]] | None = None, mode: Literal['thread', 'process'] | None=None):
+        self.actor_types = actor_types or self.actor_types or self.autodiscover_actor_types()
+        self.mode = mode or self.mode
+
+    def __repr__(self) -> str:
+        label = 'tid' if self.mode == 'thread' else 'pid'
+        return f'[underline]{self.name}[/underline]\\[{label}={self.pid}]'
+    
+    def __str__(self) -> str:
+        return self.__repr__()
+    
+    @classproperty
+    def name(cls) -> str:
+        return cls.__name__   # type: ignore
+    
+    def fork_as_thread(self):
+        self.thread = Thread(target=self.runloop)
+        self.thread.start()
+        assert self.thread.native_id is not None
+        return self.thread.native_id
+    
+    def fork_as_process(self):
+        self.process = Process(target=self.runloop)
+        self.process.start()
+        assert self.process.pid is not None
+        return self.process.pid
+
+    def start(self) -> int:
+        if self.mode == 'thread':
+            return self.fork_as_thread()
+        elif self.mode == 'process':
+            return self.fork_as_process()
+        raise ValueError(f'Invalid orchestrator mode: {self.mode}')
+    
+    @classmethod
+    def autodiscover_actor_types(cls) -> Dict[str, Type[ActorType]]:
+        # returns a Dict of all discovered {actor_type_id: ActorType} across the codebase
+        # override this method in a subclass to customize the actor types that are used
+        # return {'Snapshot': SnapshotActorType, 'ArchiveResult_chrome': ChromeActorType, ...}
+        return {
+            # look through all models and find all classes that inherit from ActorType
+            # actor_type.__name__: actor_type
+            # for actor_type in abx.pm.hook.get_all_ACTORS_TYPES().values()
+        }
+    
+    @classmethod
+    def get_orphaned_objects(cls, all_queues) -> list:
+        # returns a list of objects that are in the queues of all actor types but not in the queues of any other actor types
+        all_queued_ids = itertools.chain(*[queue.values('id', flat=True) for queue in all_queues.values()])
+        orphaned_objects = []
+        for model in apps.get_models():
+            if hasattr(model, 'retry_at'):
+                orphaned_objects.extend(model.objects.filter(retry_at__lt=timezone.now()).exclude(id__in=all_queued_ids))
+        return orphaned_objects
+    
+    def on_startup(self):
+        if self.mode == 'thread':
+            self.pid = get_native_id()
+            print(f'[green]👨‍✈️ {self}.on_startup() STARTUP (THREAD)[/green]')
+        elif self.mode == 'process':
+            self.pid = os.getpid()
+            print(f'[green]👨‍✈️ {self}.on_startup() STARTUP (PROCESS)[/green]')
+        # abx.pm.hook.on_orchestrator_startup(self)
+    
+    def on_shutdown(self, err: BaseException | None = None):
+        print(f'[grey53]👨‍✈️ {self}.on_shutdown() SHUTTING DOWN[/grey53]', err or '[green](gracefully)[/green]')
+        # abx.pm.hook.on_orchestrator_shutdown(self)
+        
+    def on_tick_started(self, all_queues):
+        # total_pending = sum(queue.count() for queue in all_queues.values())
+        # print(f'👨‍✈️ {self}.on_tick_started()', f'total_pending={total_pending}')
+        # abx.pm.hook.on_orchestrator_tick_started(self, actor_types, all_queues)
+        pass
+    
+    def on_tick_finished(self, all_queues, all_existing_actors, all_spawned_actors):
+        if all_spawned_actors:
+            total_queue_length = sum(queue.count() for queue in all_queues.values())
+            print(f'[grey53]👨‍✈️ {self}.on_tick_finished() queue={total_queue_length} existing_actors={len(all_existing_actors)} spawned_actors={len(all_spawned_actors)}[/grey53]')
+        # abx.pm.hook.on_orchestrator_tick_finished(self, actor_types, all_queues)
+
+    def on_idle(self, all_queues):
+        # print(f'👨‍✈️ {self}.on_idle()')
+        # abx.pm.hook.on_orchestrator_idle(self)
+        # check for orphaned objects left behind
+        if self.idle_count == 60:
+            orphaned_objects = self.get_orphaned_objects(all_queues)
+            if orphaned_objects:
+                print('[red]👨‍✈️ WARNING: some objects may not be processed, no actor has claimed them after 60s:[/red]', orphaned_objects)
+
+    def runloop(self):
+        self.on_startup()
+        try:
+            while True:
+                all_queues = {
+                    actor_type: actor_type.get_queue()
+                    for actor_type in self.actor_types.values()
+                }
+                if not all_queues:
+                    raise Exception('Failed to find any actor_types to process')
+
+                self.on_tick_started(all_queues)
+
+                all_existing_actors = []
+                all_spawned_actors = []
+
+                for actor_type, queue in all_queues.items():
+                    try:
+                        existing_actors = actor_type.get_running_actors()
+                        all_existing_actors.extend(existing_actors)
+                        actors_to_spawn = actor_type.get_actors_to_spawn(queue, existing_actors)
+                        for launch_kwargs in actors_to_spawn:
+                            new_actor_pid = actor_type.start(mode='process', **launch_kwargs)
+                            all_spawned_actors.append(new_actor_pid)
+                    except Exception as err:
+                        print(f'🏃‍♂️ ERROR: {self} Failed to get {actor_type} queue & running actors', err)
+                    except BaseException:
+                        raise
+
+                if not any(queue.exists() for queue in all_queues.values()):
+                    self.on_idle(all_queues)
+                    self.idle_count += 1
+                    time.sleep(1)
+                else:
+                    self.idle_count = 0
+                    
+                self.on_tick_finished(all_queues, all_existing_actors, all_spawned_actors)
+                time.sleep(1)
+
+        except BaseException as err:
+            if isinstance(err, KeyboardInterrupt):
+                print()
+            else:
+                print(f'\n[red]🏃‍♂️ {self}.runloop() FATAL:[/red]', err.__class__.__name__, err)
+            self.on_shutdown(err=err)
+
+
+
+from archivebox.config.django import setup_django
+
+setup_django()
+
+from core.models import ArchiveResult, Snapshot
+
+from django.utils import timezone
+
+from django import db
+from django.db import connection
+
+
+from crawls.actors import CrawlActor
+from .actor_snapshot import SnapshotActor
+
+from abx_plugin_singlefile.actors import SinglefileActor
+
+
+class FaviconActor(ActorType[ArchiveResult]):
+    CLAIM_ORDER: ClassVar[str] = 'created_at DESC'
+    CLAIM_WHERE: ClassVar[str] = 'status = "queued" AND extractor = "favicon"'
+    CLAIM_SET: ClassVar[str] = 'status = "started"'
+    
+    @classproperty
+    def QUERYSET(cls) -> QuerySet:
+        return ArchiveResult.objects.filter(status='failed', extractor='favicon')
+
+    def tick(self, obj: ArchiveResult):
+        print(f'[grey53]{self}.tick({obj.abid or obj.id}, status={obj.status}) remaining:[/grey53]', self.get_queue().count())
+        updated = ArchiveResult.objects.filter(id=obj.id, status='started').update(status='success') == 1
+        if not updated:
+            raise Exception(f'Failed to update {obj.abid or obj.id}, interrupted by another actor writing to the same object')
+        obj.refresh_from_db()
+        obj.save()
+
+
+class ExtractorsOrchestrator(Orchestrator):
+    actor_types = {
+        'CrawlActor': CrawlActor,
+        'SnapshotActor': SnapshotActor,
+        'FaviconActor': FaviconActor,
+        'SinglefileActor': SinglefileActor,
+    }
+
+
+if __name__ == '__main__':    
+    orchestrator = ExtractorsOrchestrator()
+    orchestrator.start()
+    
+    snap = Snapshot.objects.last()
+    assert snap is not None
+    created = 0
+    while True:
+        time.sleep(0.05)
+        # try:
+        #     ArchiveResult.objects.bulk_create([
+        #         ArchiveResult(
+        #             id=uuid.uuid4(),
+        #             snapshot=snap,
+        #             status='failed',
+        #             extractor='favicon',
+        #             cmd=['echo', '"hello"'],
+        #             cmd_version='1.0',
+        #             pwd='.',
+        #             start_ts=timezone.now(),
+        #             end_ts=timezone.now(),
+        #             created_at=timezone.now(),
+        #             modified_at=timezone.now(),
+        #             created_by_id=1,
+        #         )
+        #         for _ in range(100)
+        #     ])
+        #     created += 100
+        #     if created % 1000 == 0:
+        #         print(f'[blue]Created {created} ArchiveResults...[/blue]')
+        #         time.sleep(25)
+        # except Exception as err:
+        #     print(err)
+        #     db.connections.close_all()
+        # except BaseException as err:
+        #     print(err)
+        #     break
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.
@@ -1 +1 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'
@@ -1,120 +1,31 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from django import forms
-from django.contrib import admin
-from django.http import HttpRequest
-from django.utils.text import capfirst
-from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
-from signal_webhooks.settings import webhook_settings
-from signal_webhooks.utils import get_webhook_model, model_from_reference
+from signal_webhooks.admin import WebhookAdmin
+from signal_webhooks.utils import get_webhook_model

-from archivebox.base_models.admin import BaseModelAdmin
+from abid_utils.admin import ABIDModelAdmin

-from archivebox.api.models import APIToken
+from api.models import APIToken


-def _webhook_fields(*names: str) -> tuple[str, ...]:
-    model_fields = {field.name for field in get_webhook_model()._meta.fields}
-    return tuple(name for name in names if name in model_fields)
+class APITokenAdmin(ABIDModelAdmin):
+    list_display = ('created_at', 'abid', 'created_by', 'token_redacted', 'expires')
+    sort_fields = ('abid', 'created_at', 'created_by', 'expires')
+    readonly_fields = ('created_at', 'modified_at', 'abid_info')
+    search_fields = ('id', 'abid', 'created_by__username', 'token')
+    fields = ('created_by', 'token', 'expires', *readonly_fields)

-
-class APITokenAdmin(BaseModelAdmin):
-    list_display = ("created_at", "id", "created_by", "token_redacted", "expires")
-    sort_fields = ("id", "created_at", "created_by", "expires")
-    readonly_fields = ("created_at", "modified_at")
-    search_fields = ("id", "created_by__username", "token")
-
-    fieldsets = (
-        (
-            "Token",
-            {
-                "fields": ("token", "expires"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Owner",
-            {
-                "fields": ("created_by",),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": ("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
-    )
-
-    list_filter = ("created_by",)
-    ordering = ["-created_at"]
+    list_filter = ('created_by',)
+    ordering = ['-created_at']
    list_per_page = 100


-class OutboundWebhookAdminForm(WebhookModelForm):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.fields["ref"] = forms.ChoiceField(
-            label=self.fields["ref"].label,
-            help_text=self.fields["ref"].help_text,
-            choices=[
-                (ref, f"{capfirst(model_from_reference(ref, check_hooks=False)._meta.verbose_name_plural)} ({ref})")
-                for ref in sorted(webhook_settings.HOOKS)
-            ],
-        )
+class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
+    list_display = ('created_at', 'created_by', 'abid', *WebhookAdmin.list_display)
+    sort_fields = ('created_at', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
+    readonly_fields = ('created_at', 'modified_at', 'abid_info', *WebhookAdmin.readonly_fields)


-class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
-    form = OutboundWebhookAdminForm
-    list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
-    sort_fields = _webhook_fields("created_at", "created_by", "id", "ref", "endpoint", "last_success", "last_failure")
-    readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
-
-    fieldsets = (
-        (
-            "Webhook",
-            {
-                "fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"),
-                "classes": ("card", "wide"),
-            },
-        ),
-        (
-            "Authentication",
-            {
-                "fields": _webhook_fields("auth_token"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Status",
-            {
-                "fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Owner",
-            {
-                "fields": _webhook_fields("created_by"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": _webhook_fields("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
-    )
-
-    def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:
-        """Preserve WebhookAdmin's auth token filter with Django's current admin signature."""
-        return not lookup.startswith("auth_token") and admin.ModelAdmin.lookup_allowed(self, lookup, value, request)
-
-
-def register_admin(admin_site: admin.AdminSite) -> None:
+def register_admin(admin_site):
    admin_site.register(APIToken, APITokenAdmin)
    admin_site.register(get_webhook_model(), CustomWebhookAdmin)
@@ -1,14 +1,15 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

 from django.apps import AppConfig

+import abx
+

 class APIConfig(AppConfig):
-    name = "archivebox.api"
-    label = "api"
+    name = 'api'


+@abx.hookimpl
 def register_admin(admin_site):
-    from archivebox.api.admin import register_admin
-
+    from api.admin import register_admin
    register_admin(admin_site)
@@ -1,19 +1,21 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

+from typing import Any, Optional, cast
 from datetime import timedelta

-from django.utils import timezone
 from django.http import HttpRequest
+from django.utils import timezone
+from django.contrib.auth import login
 from django.contrib.auth import authenticate
-from django.contrib.auth.models import User
+from django.contrib.auth.models import AbstractBaseUser

-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader
+from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser
 from ninja.errors import HttpError


-def get_or_create_api_token(user: User | None):
-    from archivebox.api.models import APIToken
-
+def get_or_create_api_token(user):
+    from api.models import APIToken
+    
    if user and user.is_superuser:
        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
        if api_tokens.exists():
@@ -22,116 +24,116 @@ def get_or_create_api_token(user: User | None):
        else:
            # does not exist, create a new one
            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
-
-        if api_token is None:
-            return None
+        
        assert api_token.is_valid(), f"API token is not valid {api_token}"

        return api_token
    return None


-def auth_using_token(token: str | None, request: HttpRequest | None = None) -> User | None:
+def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
    """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
-    from archivebox.api.models import APIToken  # lazy import model to avoid loading it at urls.py import time
+    from api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
+    
+    user = None

-    user: User | None = None
-
-    submitted_empty_form = str(token).strip() in ("string", "", "None", "null")
+    submitted_empty_form = str(token).strip() in ('string', '', 'None', 'null')
    if not submitted_empty_form:
        try:
-            api_token = APIToken.objects.get(token=token)
-            if api_token.is_valid() and isinstance(api_token.created_by, User):
-                user = api_token.created_by
-                if request is not None:
-                    setattr(request, "_api_token", api_token)
+            token = APIToken.objects.get(token=token)
+            if token.is_valid():
+                user = token.created_by
+                request._api_token = token
        except APIToken.DoesNotExist:
            pass

-    return user
+    if not user:
+        # print('[❌] Failed to authenticate API user using API Key:', request)
+        return None
+    
+    return cast(AbstractBaseUser, user)

-
-def token_from_request(request: HttpRequest) -> str:
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-    return token
-
-
-def authenticated_user_from_request(request: HttpRequest) -> User | None:
-    user = request.user
-    if user.is_authenticated and user.is_active:
-        return user
-
-    token = token_from_request(request)
-    token_user = auth_using_token(token=token, request=request) if token else None
-    if token_user and token_user.is_active:
-        request.user = token_user
-        return token_user
-    return None
-
-
-def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
+def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
    """Given a username and password, check if they are valid and return the corresponding user"""
-    user: User | None = None
-
-    submitted_empty_form = (username, password) in (("string", "string"), ("", ""), (None, None))
+    user = None
+    
+    submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
    if not submitted_empty_form:
-        authenticated_user = authenticate(
+        user = authenticate(
            username=username,
            password=password,
        )
-        if isinstance(authenticated_user, User):
-            user = authenticated_user
-    return user
+
+    if not user:
+        # print('[❌] Failed to authenticate API user using API Key:', request)
+        user = None
+
+    return cast(AbstractBaseUser | None, user)


 ### Base Auth Types


-def _require_superuser(user: User | None, request: HttpRequest, auth_method: str) -> User | None:
-    if user and user.pk:
-        request.user = user
-        setattr(request, "_api_auth_method", auth_method)
-        if not user.is_superuser:
-            raise HttpError(403, "Valid credentials but User does not have permission (make sure user.is_superuser=True)")
-    return user
+class APITokenAuthCheck:
+    """The base class for authentication methods that use an api.models.APIToken"""
+    def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
+        request.user = auth_using_token(
+            token=key,
+            request=request,
+        )
+        if request.user and request.user.pk:
+            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
+            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
+            request._api_auth_method = self.__class__.__name__
+
+            if not request.user.is_superuser:
+                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
+        return request.user
+
+
+class UserPassAuthCheck:
+    """The base class for authentication methods that use a username & password"""
+    def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
+        request.user = auth_using_password(
+            username=username,
+            password=password,
+            request=request,
+        )
+        if request.user and request.user.pk:
+            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
+            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
+            request._api_auth_method = self.__class__.__name__
+
+            if not request.user.is_superuser:
+                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
+
+        return request.user


 ### Django-Ninja-Provided Auth Methods

-
-class HeaderTokenAuth(APIKeyHeader):
+class HeaderTokenAuth(APITokenAuthCheck, APIKeyHeader):
    """Allow authenticating by passing X-API-Key=xyz as a request header"""
-
    param_name = "X-ArchiveBox-API-Key"

-    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
-        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
-
-
-class BearerTokenAuth(HttpBearer):
+class BearerTokenAuth(APITokenAuthCheck, HttpBearer):
    """Allow authenticating by passing Bearer=xyz as a request header"""
+    pass

-    def authenticate(self, request: HttpRequest, token: str) -> User | None:
-        return _require_superuser(auth_using_token(token=token, request=request), request, self.__class__.__name__)
-
-
-class QueryParamTokenAuth(APIKeyQuery):
+class QueryParamTokenAuth(APITokenAuthCheck, APIKeyQuery):
    """Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
-
    param_name = "api_key"

-    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
-        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
-
+class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
+    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
+    pass

 ### Enabled Auth Methods

 API_AUTH_METHODS = [
    HeaderTokenAuth(),
    BearerTokenAuth(),
-    QueryParamTokenAuth(),
+    QueryParamTokenAuth(), 
    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
+    UsernameAndPasswordAuth(),
 ]
@@ -1,32 +0,0 @@
-__package__ = "archivebox.api"
-
-from django.http import HttpResponse
-
-
-class ApiCorsMiddleware:
-    """Attach permissive CORS headers for API routes (token-based auth)."""
-
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        if request.path.startswith("/api/"):
-            if request.method == "OPTIONS" and request.META.get("HTTP_ACCESS_CONTROL_REQUEST_METHOD"):
-                response = HttpResponse(status=204)
-                return self._add_cors_headers(request, response)
-
-            response = self.get_response(request)
-            return self._add_cors_headers(request, response)
-
-        return self.get_response(request)
-
-    def _add_cors_headers(self, request, response):
-        origin = request.META.get("HTTP_ORIGIN")
-        if not origin:
-            return response
-
-        response["Access-Control-Allow-Origin"] = "*"
-        response["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
-        response["Access-Control-Allow-Headers"] = "Authorization, X-ArchiveBox-API-Key, Content-Type, X-CSRFToken"
-        response["Access-Control-Max-Age"] = "600"
-        return response
@@ -1,239 +1,29 @@
-# Generated by hand on 2025-12-29
-# Creates APIToken and OutboundWebhook tables using raw SQL
+# Generated by Django 4.2.11 on 2024-04-25 04:19

+import api.models
+from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
-import django.utils.timezone
-from django.conf import settings
-from archivebox.uuid_compat import uuid7
-from archivebox.base_models.models import get_or_create_system_user_pk
-import archivebox.api.models
-import signal_webhooks.fields
-import signal_webhooks.utils
+import uuid


 class Migration(migrations.Migration):
+
    initial = True

    dependencies = [
-        ("auth", "0012_alter_user_first_name_max_length"),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
-        migrations.SeparateDatabaseAndState(
-            database_operations=[
-                migrations.RunSQL(
-                    sql="""
-                -- Create api_apitoken table
-                CREATE TABLE IF NOT EXISTS api_apitoken (
-                    id TEXT PRIMARY KEY NOT NULL,
-                    created_at DATETIME NOT NULL,
-                    modified_at DATETIME NOT NULL,
-
-                    token VARCHAR(32) NOT NULL UNIQUE,
-                    expires DATETIME,
-
-                    created_by_id INTEGER NOT NULL,
-
-                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
-                );
-                CREATE INDEX IF NOT EXISTS api_apitoken_created_by_id_idx ON api_apitoken(created_by_id);
-                CREATE INDEX IF NOT EXISTS api_apitoken_created_at_idx ON api_apitoken(created_at);
-                CREATE INDEX IF NOT EXISTS api_apitoken_token_idx ON api_apitoken(token);
-
-                -- Create api_outboundwebhook table
-                CREATE TABLE IF NOT EXISTS api_outboundwebhook (
-                    id TEXT PRIMARY KEY NOT NULL,
-                    created_at DATETIME NOT NULL,
-                    modified_at DATETIME NOT NULL,
-
-                    name VARCHAR(255) NOT NULL UNIQUE,
-                    signal VARCHAR(255) NOT NULL,
-                    ref VARCHAR(1024) NOT NULL,
-                    endpoint VARCHAR(2048) NOT NULL,
-                    headers TEXT NOT NULL DEFAULT '{}',
-                    auth_token TEXT NOT NULL DEFAULT '',
-                    enabled BOOLEAN NOT NULL DEFAULT 1,
-                    keep_last_response BOOLEAN NOT NULL DEFAULT 0,
-                    created DATETIME NOT NULL,
-                    updated DATETIME NOT NULL,
-                    last_response TEXT NOT NULL DEFAULT '',
-                    last_success DATETIME,
-                    last_failure DATETIME,
-
-                    created_by_id INTEGER NOT NULL,
-
-                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
-                );
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_by_id_idx ON api_outboundwebhook(created_by_id);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
-                    """,
-                    reverse_sql="""
-                DROP TABLE IF EXISTS api_outboundwebhook;
-                DROP TABLE IF EXISTS api_apitoken;
-                    """,
-                ),
-            ],
-            state_operations=[
-                migrations.CreateModel(
-                    name="APIToken",
-                    fields=[
-                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
-                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
-                        ("modified_at", models.DateTimeField(auto_now=True)),
-                        ("token", models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
-                        ("expires", models.DateTimeField(blank=True, null=True)),
-                        (
-                            "created_by",
-                            models.ForeignKey(
-                                default=get_or_create_system_user_pk,
-                                on_delete=django.db.models.deletion.CASCADE,
-                                to=settings.AUTH_USER_MODEL,
-                            ),
-                        ),
-                    ],
-                    options={
-                        "verbose_name": "API Key",
-                        "verbose_name_plural": "API Keys",
-                        "app_label": "api",
-                    },
-                ),
-                migrations.CreateModel(
-                    name="OutboundWebhook",
-                    fields=[
-                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
-                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
-                        ("modified_at", models.DateTimeField(auto_now=True)),
-                        (
-                            "name",
-                            models.CharField(db_index=True, help_text="Webhook name.", max_length=255, unique=True, verbose_name="name"),
-                        ),
-                        (
-                            "signal",
-                            models.CharField(
-                                choices=[
-                                    ("CREATE", "Create"),
-                                    ("UPDATE", "Update"),
-                                    ("DELETE", "Delete"),
-                                    ("M2M", "M2M changed"),
-                                    ("CREATE_OR_UPDATE", "Create or Update"),
-                                    ("CREATE_OR_DELETE", "Create or Delete"),
-                                    ("CREATE_OR_M2M", "Create or M2M changed"),
-                                    ("UPDATE_OR_DELETE", "Update or Delete"),
-                                    ("UPDATE_OR_M2M", "Update or M2M changed"),
-                                    ("DELETE_OR_M2M", "Delete or M2M changed"),
-                                    ("CREATE_UPDATE_OR_DELETE", "Create, Update or Delete"),
-                                    ("CREATE_UPDATE_OR_M2M", "Create, Update or M2M changed"),
-                                    ("CREATE_DELETE_OR_M2M", "Create, Delete or M2M changed"),
-                                    ("UPDATE_DELETE_OR_M2M", "Update, Delete or M2M changed"),
-                                    ("CREATE_UPDATE_DELETE_OR_M2M", "Create, Update or Delete, or M2M changed"),
-                                ],
-                                help_text="Signal the webhook fires to.",
-                                max_length=255,
-                                verbose_name="signal",
-                            ),
-                        ),
-                        (
-                            "ref",
-                            models.CharField(
-                                db_index=True,
-                                help_text="Dot import notation to the model the webhook is for.",
-                                max_length=1023,
-                                validators=[signal_webhooks.utils.model_from_reference],
-                                verbose_name="referenced model",
-                            ),
-                        ),
-                        (
-                            "endpoint",
-                            models.URLField(help_text="Target endpoint for this webhook.", max_length=2047, verbose_name="endpoint"),
-                        ),
-                        (
-                            "headers",
-                            models.JSONField(
-                                blank=True,
-                                default=dict,
-                                help_text="Headers to send with the webhook request.",
-                                validators=[signal_webhooks.utils.is_dict],
-                                verbose_name="headers",
-                            ),
-                        ),
-                        (
-                            "auth_token",
-                            signal_webhooks.fields.TokenField(
-                                blank=True,
-                                default="",
-                                help_text="Authentication token to use in an Authorization header.",
-                                max_length=8000,
-                                validators=[signal_webhooks.utils.decode_cipher_key],
-                                verbose_name="authentication token",
-                            ),
-                        ),
-                        ("enabled", models.BooleanField(default=True, help_text="Is this webhook enabled?", verbose_name="enabled")),
-                        (
-                            "keep_last_response",
-                            models.BooleanField(
-                                default=False,
-                                help_text="Should the webhook keep a log of the latest response it got?",
-                                verbose_name="keep last response",
-                            ),
-                        ),
-                        (
-                            "created",
-                            models.DateTimeField(auto_now_add=True, help_text="When the webhook was created.", verbose_name="created"),
-                        ),
-                        (
-                            "updated",
-                            models.DateTimeField(auto_now=True, help_text="When the webhook was last updated.", verbose_name="updated"),
-                        ),
-                        (
-                            "last_response",
-                            models.CharField(
-                                blank=True,
-                                default="",
-                                help_text="Latest response to this webhook.",
-                                max_length=8000,
-                                verbose_name="last response",
-                            ),
-                        ),
-                        (
-                            "last_success",
-                            models.DateTimeField(
-                                default=None,
-                                help_text="When the webhook last succeeded.",
-                                null=True,
-                                verbose_name="last success",
-                            ),
-                        ),
-                        (
-                            "last_failure",
-                            models.DateTimeField(
-                                default=None,
-                                help_text="When the webhook last failed.",
-                                null=True,
-                                verbose_name="last failure",
-                            ),
-                        ),
-                        (
-                            "created_by",
-                            models.ForeignKey(
-                                default=get_or_create_system_user_pk,
-                                on_delete=django.db.models.deletion.CASCADE,
-                                to=settings.AUTH_USER_MODEL,
-                            ),
-                        ),
-                    ],
-                    options={
-                        "verbose_name": "API Outbound Webhook",
-                        "app_label": "api",
-                    },
-                ),
-                migrations.AddConstraint(
-                    model_name="outboundwebhook",
-                    constraint=models.UniqueConstraint(fields=["ref", "endpoint"], name="prevent_duplicate_hooks_api_outboundwebhook"),
-                ),
+        migrations.CreateModel(
+            name='APIToken',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('token', models.CharField(default=api.models.generate_secret_token, max_length=32, unique=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('expires', models.DateTimeField(blank=True, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
            ],
        ),
    ]
@@ -0,0 +1,17 @@
+# Generated by Django 5.0.4 on 2024-04-26 05:28
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='apitoken',
+            options={'verbose_name': 'API Key', 'verbose_name_plural': 'API Keys'},
+        ),
+    ]
@@ -0,0 +1,77 @@
+# Generated by Django 5.0.6 on 2024-06-03 01:52
+
+import abid_utils.models
+import charidfield.fields
+import django.db.models.deletion
+import signal_webhooks.fields
+import signal_webhooks.utils
+import uuid
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0002_alter_apitoken_options'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='apitoken',
+            old_name='user',
+            new_name='created_by',
+        ),
+        migrations.AddField(
+            model_name='apitoken',
+            name='abid',
+            field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='apt_', unique=True),
+        ),
+        migrations.AddField(
+            model_name='apitoken',
+            name='modified',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name='apitoken',
+            name='uuid',
+            field=models.UUIDField(blank=True, null=True, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False),
+        ),
+        migrations.CreateModel(
+            name='OutboundWebhook',
+            fields=[
+                ('name', models.CharField(db_index=True, help_text='Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).', max_length=255, unique=True, verbose_name='name')),
+                ('signal', models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='The type of event the webhook should fire for (e.g. Create, Update, Delete).', max_length=255, verbose_name='signal')),
+                ('ref', models.CharField(db_index=True, help_text='Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model')),
+                ('endpoint', models.URLField(help_text='External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).', max_length=2047, verbose_name='endpoint')),
+                ('headers', models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers')),
+                ('auth_token', signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token')),
+                ('enabled', models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled')),
+                ('keep_last_response', models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response')),
+                ('updated', models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated')),
+                ('last_response', models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response')),
+                ('last_success', models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success')),
+                ('last_failure', models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('modified', models.DateTimeField(auto_now=True)),
+                ('id', models.UUIDField(blank=True, null=True, unique=True)),
+                ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='whk_', unique=True)),
+                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'verbose_name': 'API Outbound Webhook',
+                'abstract': False,
+            },
+        ),
+        migrations.AddConstraint(
+            model_name='outboundwebhook',
+            constraint=models.UniqueConstraint(fields=('ref', 'endpoint'), name='prevent_duplicate_hooks_api_outboundwebhook'),
+        ),
+    ]
@@ -0,0 +1,24 @@
+# Generated by Django 5.1 on 2024-08-20 10:44
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0003_rename_user_apitoken_created_by_apitoken_abid_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='uuid',
+            field=models.UUIDField(blank=True, editable=False, null=True, unique=True),
+        ),
+    ]
@@ -0,0 +1,22 @@
+# Generated by Django 5.1 on 2024-08-20 22:40
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0004_alter_apitoken_id_alter_apitoken_uuid'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='apitoken',
+            name='uuid',
+        ),
+        migrations.RemoveField(
+            model_name='outboundwebhook',
+            name='id',
+        ),
+    ]
@@ -0,0 +1,29 @@
+# Generated by Django 5.1 on 2024-08-20 22:43
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='outboundwebhook',
+            old_name='uuid',
+            new_name='id'
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+    ]
@@ -0,0 +1,22 @@
+# Generated by Django 5.1 on 2024-08-20 22:52
+
+import abid_utils.models
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0006_remove_outboundwebhook_uuid_apitoken_id_and_more'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='apitoken',
+            name='created_by',
+            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+    ]
@@ -0,0 +1,47 @@
+# Generated by Django 5.1 on 2024-09-04 23:32
+
+import abid_utils.models
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0007_alter_apitoken_created_by'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='apitoken',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='created_by',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='created',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='created_by',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='id',
+            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+    ]
@@ -0,0 +1,39 @@
+# Generated by Django 5.1 on 2024-09-05 00:26
+
+import abid_utils.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0008_alter_apitoken_created_alter_apitoken_created_by_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='apitoken',
+            old_name='created',
+            new_name='created_at',
+        ),
+        migrations.RenameField(
+            model_name='apitoken',
+            old_name='modified',
+            new_name='modified_at',
+        ),
+        migrations.RenameField(
+            model_name='outboundwebhook',
+            old_name='modified',
+            new_name='modified_at',
+        ),
+        migrations.AddField(
+            model_name='outboundwebhook',
+            name='created_at',
+            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, help_text='When the webhook was created.', verbose_name='created'),
+        ),
+    ]
@@ -1,54 +1,127 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

 import secrets
-from archivebox.uuid_compat import CompactUUIDField, uuid7
+from datetime import timedelta

 from django.conf import settings
 from django.db import models
 from django.utils import timezone
-from django_stubs_ext.db.models import TypedModelMeta
+
 from signal_webhooks.models import WebhookBase

-from archivebox.base_models.models import get_or_create_system_user_pk
+from django_stubs_ext.db.models import TypedModelMeta
+
+from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
+


 def generate_secret_token() -> str:
+    # returns cryptographically secure string with len() == 32
    return secrets.token_hex(16)


-class APIToken(models.Model):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
+class APIToken(ABIDModel):
+    """
+    A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox.
+    """
+    # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
+    abid_prefix = 'apt_'
+    abid_ts_src = 'self.created_at'
+    abid_uri_src = 'self.created_by_id'
+    abid_subtype_src = '"01"'
+    abid_rand_src = 'self.id'
+    abid_drift_allowed = True
+
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)
+
    token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
    expires = models.DateTimeField(null=True, blank=True)

    class Meta(TypedModelMeta):
-        app_label = "api"
        verbose_name = "API Key"
        verbose_name_plural = "API Keys"

    def __str__(self) -> str:
        return self.token

+    def __repr__(self) -> str:
+        return f'<APIToken user={self.created_by.username} token={self.token_redacted}>'
+
+    def __json__(self) -> dict:
+        return {
+            "TYPE":             "APIToken",    
+            "id":               str(self.pk),
+            "abid":             str(self.ABID),
+            "created_by_id":    str(self.created_by_id),
+            "token":            self.token,
+            "created_at":       self.created_at.isoformat(),
+            "expires":          self.expires_as_iso8601,
+        }
+
+    @property
+    def expires_as_iso8601(self):
+        """Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
+        expiry_date = self.expires or (timezone.now() + timedelta(days=365 * 100))
+
+        return expiry_date.isoformat()
+    
    @property
    def token_redacted(self):
-        return f"************{self.token[-4:]}"
+        return f'************{self.token[-4:]}'

    def is_valid(self, for_date=None):
-        return not self.expires or self.expires >= (for_date or timezone.now())
+        for_date = for_date or timezone.now()
+
+        if self.expires and self.expires < for_date:
+            return False
+
+        return True


-class OutboundWebhook(WebhookBase):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
+
+
+
+
+# monkey patch django-signals-webhooks to change how it shows up in Admin UI
+
+class OutboundWebhook(ABIDModel, WebhookBase):
+    """
+    Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using:
+        settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
+    """
+    abid_prefix = 'whk_'
+    abid_ts_src = 'self.created_at'
+    abid_uri_src = 'self.endpoint'
+    abid_subtype_src = 'self.ref'
+    abid_rand_src = 'self.id'
+    abid_drift_allowed = True
+
+    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
+    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)

+    # More fields here: WebhookBase...
+
+    WebhookBase._meta.get_field('name').help_text = (
+        'Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).')
+    WebhookBase._meta.get_field('signal').help_text = (
+        'The type of event the webhook should fire for (e.g. Create, Update, Delete).')
+    WebhookBase._meta.get_field('ref').help_text = (
+        'Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).')
+    WebhookBase._meta.get_field('endpoint').help_text = (
+        'External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).')
+
    class Meta(WebhookBase.Meta):
-        app_label = "api"
-        verbose_name = "API Outbound Webhook"
+        verbose_name = 'API Outbound Webhook'
+

    def __str__(self) -> str:
-        return f"[{self.id}] {self.ref} -> {self.endpoint}"
+        return f'[{self.abid}] {self.ref} -> {self.endpoint}'
@@ -0,0 +1,30 @@
+__package__ = 'archivebox.api'
+
+# from django.test import TestCase
+# from ninja.testing import TestClient
+
+# from .routes_cli import router
+
+# class ArchiveBoxCLIAPITestCase(TestCase):
+#     def setUp(self):
+#         self.client = TestClient(router)
+
+#     def test_add_endpoint(self):
+#         response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
+
+#     def test_remove_endpoint(self):
+#         response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
+
+#     def test_update_endpoint(self):
+#         response = self.client.post("/update", json={})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
+
+#     def test_list_all_endpoint(self):
+#         response = self.client.post("/list_all", json={})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
@@ -1,28 +1,16 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from urllib.parse import quote
-
-from django.http import HttpRequest, HttpResponseRedirect
-from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView

-from archivebox.core.routes_util import build_web_url
 from .v1_api import urls as v1_api_urls

-
-def archive_redirect_view(request: HttpRequest, url: str) -> HttpResponseRedirect:
-    if request.META.get("QUERY_STRING"):
-        url = f"{url}?{request.META['QUERY_STRING']}"
-    return redirect(build_web_url(f"/web/{quote(url, safe=':/')}", request=request))
-
-
 urlpatterns = [
-    path("", RedirectView.as_view(url="/api/v1/docs")),
-    path("archive/<path:url>", archive_redirect_view, name="api-archive-redirect"),
-    path("v1/", RedirectView.as_view(url="/api/v1/docs")),
-    path("v1/", v1_api_urls),
-    path("v1", RedirectView.as_view(url="/api/v1/docs")),
+    path("",                 RedirectView.as_view(url='/api/v1')),
+
+    path("v1/",              v1_api_urls),
+    path("v1",               RedirectView.as_view(url='/api/v1/docs')),
+
    # ... v2 can be added here ...
    # path("v2/",              v2_api_urls),
    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
@@ -1,4 +1,4 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'


 from io import StringIO
@@ -7,7 +7,6 @@ from contextlib import redirect_stdout, redirect_stderr

 from django.http import HttpRequest, HttpResponse
 from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
-from django.contrib.auth.models import User

 from ninja import NinjaAPI, Swagger

@@ -16,13 +15,12 @@ from ninja import NinjaAPI, Swagger
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH

-from archivebox.api.auth import API_AUTH_METHODS
-from archivebox.api.models import APIToken
+from api.auth import API_AUTH_METHODS


-COMMIT_HASH = get_COMMIT_HASH() or "unknown"
+COMMIT_HASH = get_COMMIT_HASH() or 'unknown'

-html_description = f"""
+html_description=f'''
 <h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
 <br/>
 <i><b>WARNING: This API is still in an early development stage and may change!</b></i>
@@ -35,48 +33,44 @@ html_description = f"""
 <li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
 </ul>
 <small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
-"""
+'''


 def register_urls(api: NinjaAPI) -> NinjaAPI:
-    api.add_router("/auth/", "archivebox.api.v1_auth.router")
-    api.add_router("/core/", "archivebox.api.v1_core.router")
-    api.add_router("/crawls/", "archivebox.api.v1_crawls.router")
-    api.add_router("/cli/", "archivebox.api.v1_cli.router")
-    api.add_router("/machine/", "archivebox.api.v1_machine.router")
-    api.add_router("/personas/", "archivebox.api.v1_personas.router")
+    api.add_router('/auth/',     'api.v1_auth.router')
+    api.add_router('/core/',     'api.v1_core.router')
+    api.add_router('/cli/',      'api.v1_cli.router')
    return api


-class NinjaAPIWithIOCapture(NinjaAPI):
+class NinjaAPIWithIOCapture(NinjaAPI):    
    def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
        stdout, stderr = StringIO(), StringIO()

        with redirect_stderr(stderr):
            with redirect_stdout(stdout):
-                setattr(request, "stdout", stdout)
-                setattr(request, "stderr", stderr)
+                request.stdout = stdout
+                request.stderr = stderr

                response = super().create_temporal_response(request)

-        # Disable caching of API responses entirely
-        response["Cache-Control"] = "no-store"
+        # Diable caching of API responses entirely
+        response['Cache-Control'] = 'no-store'

        # Add debug stdout and stderr headers to response
-        response["X-ArchiveBox-Stdout"] = stdout.getvalue().replace("\n", "\\n")[:200]
-        response["X-ArchiveBox-Stderr"] = stderr.getvalue().replace("\n", "\\n")[:200]
+        response['X-ArchiveBox-Stdout'] = str(request.stdout)[200:]
+        response['X-ArchiveBox-Stderr'] = str(request.stderr)[200:]
        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'

        # Add Auth Headers to response
-        api_token_attr = request.__dict__.get("_api_token")
-        api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
-        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
+        api_token = getattr(request, '_api_token', None)
+        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else 'Never'

-        response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
-        response["X-ArchiveBox-Auth-Expires"] = token_expiry
-        response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
-        response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
+        response['X-ArchiveBox-Auth-Method'] = getattr(request, '_api_auth_method', None) or 'None'
+        response['X-ArchiveBox-Auth-Expires'] = token_expiry
+        response['X-ArchiveBox-Auth-Token-Id'] = api_token.abid if api_token else 'None'
+        response['X-ArchiveBox-Auth-User-Id'] = request.user.pk if request.user.pk else 'None'
+        response['X-ArchiveBox-Auth-User-Username'] = request.user.username if request.user.pk else 'None'

        # import ipdb; ipdb.set_trace()
        # print('RESPONDING NOW', response)
@@ -85,9 +79,10 @@ class NinjaAPIWithIOCapture(NinjaAPI):


 api = NinjaAPIWithIOCapture(
-    title="ArchiveBox API",
+    title='ArchiveBox API',
    description=html_description,
-    version=VERSION,
+    version='1.0.0',
+    csrf=False,
    auth=API_AUTH_METHODS,
    urls_namespace="api-1",
    docs=Swagger(settings={"persistAuthorization": True}),
@@ -104,15 +99,15 @@ def generic_exception_handler(request, err):
    if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
        status = 404

-    print("".join(format_exception(err)))
+    print(''.join(format_exception(err)))

    return api.create_response(
        request,
        {
            "succeeded": False,
-            "message": f"{err.__class__.__name__}: {err}",
+            "message": f'{err.__class__.__name__}: {err}',
            "errors": [
-                "".join(format_exception(err)),
+                ''.join(format_exception(err)),
                # or send simpler parent-only traceback:
                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
            ],
@@ -121,6 +116,7 @@ def generic_exception_handler(request, err):
    )


+
 # import orjson
 # from ninja.renderers import BaseRenderer
 # class ORJSONRenderer(BaseRenderer):
@@ -1,28 +1,26 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from django.http import HttpRequest
+from typing import Optional

 from ninja import Router, Schema
+from django.utils import timezone
+from datetime import timedelta

-from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token
+from api.models import APIToken
+from api.auth import auth_using_token, auth_using_password, get_or_create_api_token


-router = Router(tags=["Authentication"], auth=None)
+router = Router(tags=['Authentication'], auth=None)


 class PasswordAuthSchema(Schema):
    """Schema for a /get_api_token request"""
-
-    username: str | None = None
-    password: str | None = None
+    username: Optional[str] = None
+    password: Optional[str] = None


-@router.post(
-    "/get_api_token",
-    auth=None,
-    summary="Generate an API token for a given username & password (or currently logged-in user)",
-)  # auth=None because they are not authed yet
-def get_api_token(request: HttpRequest, auth_data: PasswordAuthSchema):
+@router.post("/get_api_token", auth=None, summary='Generate an API token for a given username & password (or currently logged-in user)')             # auth=None because they are not authed yet
+def get_api_token(request, auth_data: PasswordAuthSchema):
    user = auth_using_password(
        username=auth_data.username,
        password=auth_data.password,
@@ -32,34 +30,24 @@ def get_api_token(request: HttpRequest, auth_data: PasswordAuthSchema):
    if user and user.is_superuser:
        api_token = get_or_create_api_token(user)
        assert api_token is not None, "Failed to create API token"
-        return {
-            "success": True,
-            "user_id": str(user.pk),
-            "username": user.username,
-            "token": api_token.token,
-            "expires": api_token.expires.isoformat() if api_token.expires else None,
-        }
-
+        return api_token.__json__()
+    
    return {"success": False, "errors": ["Invalid credentials"]}


+
 class TokenAuthSchema(Schema):
    """Schema for a /check_api_token request"""
-
    token: str


-@router.post(
-    "/check_api_token",
-    auth=None,
-    summary="Validate an API token to make sure its valid and non-expired",
-)  # auth=None because they are not authed yet
-def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
+@router.post("/check_api_token", auth=None, summary='Validate an API token to make sure its valid and non-expired')        # auth=None because they are not authed yet
+def check_api_token(request, token_data: TokenAuthSchema):
    user = auth_using_token(
        token=token_data.token,
        request=request,
    )
    if user:
        return {"success": True, "user_id": str(user.pk)}
-
+    
    return {"success": False, "user_id": None}
@@ -1,322 +1,236 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-import json
-from io import StringIO
-from typing import Any
+from typing import List, Dict, Any, Optional
 from enum import Enum

-from django.http import HttpRequest
-
 from ninja import Router, Schema
-from ninja.errors import HttpError
-from pydantic import Field

+from ..main import (
+    add,
+    remove,
+    update,
+    list_all,
+    schedule,
+)
 from archivebox.misc.util import ansi_to_html
-from archivebox.core.models import SnapshotQuerySet
+from archivebox.config.common import ARCHIVING_CONFIG


-# from .auth import API_AUTH_METHODS
+from .auth import API_AUTH_METHODS

 # router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=["ArchiveBox CLI Sub-Commands"])
+router = Router(tags=['ArchiveBox CLI Sub-Commands'], auth=API_AUTH_METHODS)


 # Schemas

-JSONType = list[Any] | dict[str, Any] | bool | int | str | None
-FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
-
+JSONType = List[Any] | Dict[str, Any] | bool | int | str | None

 class CLICommandResponseSchema(Schema):
    success: bool
-    errors: list[str]
+    errors: List[str]
    result: JSONType
-    result_format: str = "str"
    stdout: str
    stderr: str

+class FilterTypeChoices(str, Enum):
+    exact = 'exact'
+    substring = 'substring'
+    regex = 'regex'
+    domain = 'domain'
+    tag = 'tag'
+    timestamp = 'timestamp'

-FilterTypeChoices = Enum(
-    "FilterTypeChoices",
-    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
-    type=str,
-)
+class StatusChoices(str, Enum):
+    indexed = 'indexed'
+    archived = 'archived'
+    unarchived = 'unarchived'
+    present = 'present'
+    valid = 'valid'
+    invalid = 'invalid'
+    duplicate = 'duplicate'
+    orphaned = 'orphaned'
+    corrupted = 'corrupted'
+    unrecognized = 'unrecognized'


 class AddCommandSchema(Schema):
-    urls: list[str]
-    snapshot_ids: list[str] | None = None
+    urls: List[str]
    tag: str = ""
    depth: int = 0
-    max_urls: int = 0
-    crawl_max_size: int = 0
-    crawl_timeout: int = 0
-    snapshot_max_size: int = 0
-    parser: str = "auto"
-    plugins: str = ""
-    persona: str = "Default"
-    only_new: bool | None = None
-    update: bool = False
+    update: bool = not ARCHIVING_CONFIG.ONLY_NEW  # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW
+    update_all: bool = False
+    index_only: bool = False
    overwrite: bool = False
+    init: bool = False
+    extractors: str = ""
+    parser: str = "auto"
+
+class UpdateCommandSchema(Schema):
+    resume: Optional[float] = 0
+    only_new: bool = ARCHIVING_CONFIG.ONLY_NEW
    index_only: bool = False
-
-
-class SnapshotFilterCommandSchema(Schema):
-    after: float | None = 0
-    before: float | None = None
-    filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
-    status: str | None = None
-    url__icontains: str | None = None
-    url__istartswith: str | None = None
-    tag: str | None = None
-    crawl_id: str | None = None
-    limit: int | None = None
-    sort: str | None = None
-    search: str | None = None
-
-
-class UpdateCommandSchema(SnapshotFilterCommandSchema):
-    resume: str | None = None
-    batch_size: int = 100
-    continuous: bool = False
-    index_only: bool = False
-    migrate_only: bool = False
-
+    overwrite: bool = False
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
+    status: Optional[StatusChoices] = StatusChoices.unarchived
+    filter_type: Optional[str] = FilterTypeChoices.substring
+    filter_patterns: Optional[List[str]] = ['https://example.com']
+    extractors: Optional[str] = ""

 class ScheduleCommandSchema(Schema):
-    import_path: str | None = None
+    import_path: Optional[str] = None
    add: bool = False
-    show: bool = False
-    foreground: bool = False
-    run_all: bool = False
-    quiet: bool = False
-    every: str | None = None
-    tag: str = ""
+    every: Optional[str] = None
+    tag: str = ''
    depth: int = 0
-    only_new: bool | None = None
-    update: bool = False
    overwrite: bool = False
+    update: bool = not ARCHIVING_CONFIG.ONLY_NEW
    clear: bool = False

-
-class ListCommandSchema(SnapshotFilterCommandSchema):
+class ListCommandSchema(Schema):
+    filter_patterns: Optional[List[str]] = ['https://example.com']
+    filter_type: str = FilterTypeChoices.substring
+    status: Optional[StatusChoices] = StatusChoices.indexed
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
+    sort: str = 'bookmarked_at'
    as_json: bool = True
    as_html: bool = False
-    as_csv: str | None = "timestamp,url"
+    as_csv: str | bool = 'timestamp,url'
    with_headers: bool = False

-
-class RemoveCommandSchema(SnapshotFilterCommandSchema):
+class RemoveCommandSchema(Schema):
+    delete: bool = True
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
    filter_type: str = FilterTypeChoices.exact
-    timeout: float = 60.0
+    filter_patterns: Optional[List[str]] = ['https://example.com']


-def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
-    kwargs = args.dict()
-    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
-    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
-    return kwargs


-@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
-def cli_add(request: HttpRequest, args: AddCommandSchema):
-    from archivebox.cli.archivebox_add import add
-    from archivebox.misc.util import validate_url

-    config_overrides: dict[str, object] = {}
-    if args.only_new is not None:
-        config_overrides["ONLY_NEW"] = bool(args.only_new)
-    if args.update or args.overwrite:
-        config_overrides["ONLY_NEW"] = False
-    submitted_urls: str | list[str] = args.urls
-    if len(args.urls) == 1:
-        try:
-            validate_url(args.urls[0])
-        except ValueError:
-            submitted_urls = args.urls[0]
-    crawl, snapshots = add(
-        urls=submitted_urls,
-        snapshot_ids=args.snapshot_ids,
+@router.post("/add", response=CLICommandResponseSchema, summary='archivebox add [args] [urls]')
+def cli_add(request, args: AddCommandSchema):
+    result = add(
+        urls=args.urls,
        tag=args.tag,
        depth=args.depth,
-        max_urls=args.max_urls,
-        crawl_max_size=args.crawl_max_size,
-        crawl_timeout=args.crawl_timeout,
-        snapshot_max_size=args.snapshot_max_size,
+        update=args.update,
+        update_all=args.update_all,
        index_only=args.index_only,
-        plugins=args.plugins,
-        persona=args.persona,
+        overwrite=args.overwrite,
+        init=args.init,
+        extractors=args.extractors,
        parser=args.parser,
-        bg=True,  # Always run in background for API calls
-        created_by_id=request.user.pk,
-        config=config_overrides or None,
    )

-    snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
-    result_payload = {
-        "crawl_id": str(crawl.id),
-        "num_snapshots": len(snapshot_ids),
-        "snapshot_ids": snapshot_ids,
-        "queued_urls": args.urls,
-    }
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-
-    return {
-        "success": True,
-        "errors": [],
-        "result": result_payload,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
-def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
-    from archivebox.core.snapshot_status import normalize_snapshot_status
-
-    try:
-        status = normalize_snapshot_status(args.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
-    update_kwargs["status"] = status
-    update_kwargs["stop_daemon_stack"] = False
-
-    is_filtered_update = any(
-        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
-    )
-    matched_snapshot_ids = []
-    if is_filtered_update:
-        matched_snapshot_ids = [
-            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
-        ]
-
-    update(**update_kwargs)
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-    return {
-        "success": True,
-        "errors": [],
-        "result": {
-            "matched_count": len(matched_snapshot_ids),
-            "snapshot_ids": matched_snapshot_ids,
-        }
-        if is_filtered_update
-        else None,
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
-def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
-    from archivebox.cli.archivebox_schedule import schedule
-
-    config_overrides: dict[str, object] = {}
-    if args.only_new is not None:
-        config_overrides["ONLY_NEW"] = bool(args.only_new)
-    if args.update or args.overwrite:
-        config_overrides["ONLY_NEW"] = False
-    result = schedule(
-        import_path=args.import_path,
-        add=args.add,
-        show=args.show,
-        foreground=args.foreground,
-        run_all=args.run_all,
-        quiet=args.quiet,
-        clear=args.clear,
-        every=args.every,
-        tag=args.tag,
-        depth=args.depth,
-        config=config_overrides or None,
-    )
-
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
    return {
        "success": True,
        "errors": [],
        "result": result,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
    }


-@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
-def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
+@router.post("/update", response=CLICommandResponseSchema, summary='archivebox update [args] [filter_patterns]')
+def cli_update(request, args: UpdateCommandSchema):
+    result = update(
+        resume=args.resume,
+        only_new=args.only_new,
+        index_only=args.index_only,
+        overwrite=args.overwrite,
+        before=args.before,
+        after=args.after,
+        status=args.status,
+        filter_type=args.filter_type,
+        filter_patterns=args.filter_patterns,
+        extractors=args.extractors,
+    )
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }

-    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
-    as_json = search_kwargs.pop("as_json")
-    as_html = search_kwargs.pop("as_html")
-    as_csv = search_kwargs.pop("as_csv")
-    with_headers = search_kwargs.pop("with_headers")
-    try:
-        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err

-    result_format = "txt"
-    if as_json:
+@router.post("/schedule", response=CLICommandResponseSchema, summary='archivebox schedule [args] [import_path]')
+def cli_schedule(request, args: ScheduleCommandSchema):
+    result = schedule(
+        import_path=args.import_path,
+        add=args.add,
+        show=args.show,
+        clear=args.clear,
+        every=args.every,
+        tag=args.tag,
+        depth=args.depth,
+        overwrite=args.overwrite,
+        update=args.update,
+    )
+
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }
+
+
+
+@router.post("/list", response=CLICommandResponseSchema, summary='archivebox list [args] [filter_patterns]')
+def cli_list(request, args: ListCommandSchema):
+    result = list_all(
+        filter_patterns=args.filter_patterns,
+        filter_type=args.filter_type,
+        status=args.status,
+        after=args.after,
+        before=args.before,
+        sort=args.sort,
+        csv=args.as_csv,
+        json=args.as_json,
+        html=args.as_html,
+        with_headers=args.with_headers,
+    )
+
+    result_format = 'txt'
+    if args.as_json:
        result_format = "json"
-        result = [
-            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
-            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
-        ]
-    elif as_html:
+    elif args.as_html:
        result_format = "html"
-        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
-    elif as_csv:
+    elif args.as_csv:
        result_format = "csv"
-        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
-        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
-        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
-    else:
-        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))

-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
    return {
        "success": True,
        "errors": [],
        "result": result,
        "result_format": result_format,
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
    }
+    


-@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
-def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
-    from archivebox.cli.archivebox_remove import remove
-    from archivebox.core.models import Snapshot
-
-    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
-    timeout_arg = remove_kwargs.pop("timeout")
-    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
-    try:
-        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
+@router.post("/remove", response=CLICommandResponseSchema, summary='archivebox remove [args] [filter_patterns]')
+def cli_remove(request, args: RemoveCommandSchema):
    result = remove(
-        yes=True,  # no way to interactively ask for confirmation via API, so we force yes
-        snapshots=snapshots_to_remove,
-        timeout=timeout,
+        yes=True,            # no way to interactively ask for confirmation via API, so we force yes
+        delete=args.delete,
+        before=args.before,
+        after=args.after,
+        filter_type=args.filter_type,
+        filter_patterns=args.filter_patterns,
    )
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
    return {
-        "success": bool(result["success"]),
-        "errors": [str(result["error"])] if result["error"] else [],
+        "success": True,
+        "errors": [],
        "result": result,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
    }
+    
@@ -1,269 +0,0 @@
-__package__ = "archivebox.api"
-
-import json
-from pathlib import Path
-from uuid import UUID
-from datetime import datetime
-from django.http import FileResponse, HttpRequest
-from django.shortcuts import redirect
-from django.utils import timezone
-
-from django.contrib.auth import get_user_model
-from django.contrib.auth.models import User
-
-from ninja import Router, Schema
-from ninja.errors import HttpError
-
-from archivebox.core.models import Snapshot
-from archivebox.core.permissions import (
-    PERMISSIONS_PUBLIC,
-    PERMISSIONS_UNLISTED,
-    is_admin_user,
-    normalize_permissions,
-)
-from archivebox.config.common import get_config
-from archivebox.crawls.models import Crawl
-from archivebox.misc.util import filter_queryset_by_uuid_substring
-
-from .auth import API_AUTH_METHODS, authenticated_user_from_request
-
-router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
-
-
-class CrawlSchema(Schema):
-    TYPE: str = "crawls.models.Crawl"
-
-    id: UUID
-
-    modified_at: datetime
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-
-    status: str
-    retry_at: datetime | None
-    is_paused: bool
-
-    urls: str
-    max_depth: int
-    tags_str: str
-    config: dict
-
-    # snapshots: List[SnapshotSchema]
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-
-    @staticmethod
-    def resolve_created_by_username(obj):
-        user_model = get_user_model()
-        user = user_model.objects.get(id=obj.created_by_id)
-        username = user.username
-        return username if isinstance(username, str) else str(user)
-
-    @staticmethod
-    def resolve_config(obj):
-        # Redact credential values so REST responses can never leak the raw
-        # token/secret/api-key that the operator stored in Crawl.config.
-        from archivebox.config.common import redact_sensitive_config
-
-        return redact_sensitive_config(obj.config)
-
-    @staticmethod
-    def resolve_snapshots(obj, context):
-        if bool(context["request"].__dict__.get("with_snapshots", False)):
-            return obj.snapshot_set.all().distinct()
-        return Snapshot.objects.none()
-
-
-class CrawlUpdateSchema(Schema):
-    action: str | None = None
-    status: str | None = None
-    retry_at: datetime | None = None
-    tags: list[str] | None = None
-    tags_str: str | None = None
-
-
-class CrawlCreateSchema(Schema):
-    urls: list[str]
-    max_depth: int = 0
-    tags: list[str] | None = None
-    tags_str: str = ""
-    label: str = ""
-    notes: str = ""
-    config: dict = {}
-
-
-class CrawlDeleteResponseSchema(Schema):
-    success: bool
-    crawl_id: str
-    deleted_count: int
-    deleted_snapshots: int
-
-
-def normalize_tag_list(tags: list[str] | None = None, tags_str: str = "") -> list[str]:
-    if tags is not None:
-        return [tag.strip() for tag in tags if tag and tag.strip()]
-    return [tag.strip() for tag in tags_str.split(",") if tag.strip()]
-
-
-@router.get("/crawls", response=list[CrawlSchema], url_name="get_crawls")
-def get_crawls(request: HttpRequest):
-    return Crawl.objects.all().distinct()
-
-
-@router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
-def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
-    urls = [url.strip() for url in data.urls if url and url.strip()]
-    if not urls:
-        raise HttpError(400, "At least one URL is required")
-    if data.max_depth not in (0, 1, 2, 3, 4):
-        raise HttpError(400, "max_depth must be between 0 and 4")
-
-    tags = normalize_tag_list(data.tags, data.tags_str)
-    config = dict(data.config or {})
-    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
-    crawl = Crawl.objects.create(
-        urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls),
-        max_depth=data.max_depth,
-        tags_str=",".join(tags),
-        label=data.label,
-        notes=data.notes,
-        config=config,
-        status=Crawl.StatusChoices.QUEUED,
-        retry_at=timezone.now(),
-        created_by=request.user if isinstance(request.user, User) else None,
-    )
-    return crawl
-
-
-def get_crawl_by_ref(crawl_id: str):
-    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
-
-
-@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
-def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
-    """Get a specific Crawl by id."""
-    setattr(request, "with_snapshots", with_snapshots)
-    setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = get_crawl_by_ref(crawl_id)
-
-    if crawl and as_rss:
-        query = request.GET.copy()
-        query.pop("as_rss", None)
-        query["crawl_id"] = str(crawl.id)
-        return redirect(f"/api/v1/core/snapshots.rss?{query.urlencode()}")
-
-    return crawl
-
-
-def crawl_file(request: HttpRequest, crawl_id: str, path: str):
-    # Try to resolve the crawl first; if it doesn't exist, return 404.
-    try:
-        crawl = get_crawl_by_ref(crawl_id)
-    except Crawl.DoesNotExist:
-        raise HttpError(404, "Crawl not found")
-
-    user = authenticated_user_from_request(request)
-
-    # Gate access using the same model as SnapshotView/can_view_snapshot:
-    # admins always pass; owners can see their own crawls; otherwise the crawl
-    # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
-    if not is_admin_user(request):
-        permissions = normalize_permissions(crawl.permissions)
-        is_owner = bool(user and crawl.created_by_id == user.id)
-        if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
-            raise HttpError(404, "Crawl not found")
-
-    crawl_root = Path(crawl.output_dir).resolve()
-    file_path = (crawl_root / path).resolve()
-    if not file_path.is_file() or crawl_root not in file_path.parents:
-        raise HttpError(404, "Crawl file not found")
-
-    response = FileResponse(file_path.open("rb"))
-    response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
-    response["Pragma"] = "no-cache"
-    response["Expires"] = "0"
-    response["X-Content-Type-Options"] = "nosniff"
-    return response
-
-
-@router.get("/crawl/{crawl_id}/files/{filename}", auth=None, url_name="crawl_file_root")
-def crawl_file_root(request: HttpRequest, crawl_id: str, filename: str):
-    return crawl_file(request, crawl_id, filename)
-
-
-@router.get("/crawl/{crawl_id}/files/{folder}/{filename}", auth=None, url_name="crawl_file_nested_1")
-def crawl_file_nested_1(request: HttpRequest, crawl_id: str, folder: str, filename: str):
-    return crawl_file(request, crawl_id, f"{folder}/{filename}")
-
-
-@router.get("/crawl/{crawl_id}/files/{folder}/{subfolder}/{filename}", auth=None, url_name="crawl_file_nested_2")
-def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfolder: str, filename: str):
-    return crawl_file(request, crawl_id, f"{folder}/{subfolder}/{filename}")
-
-
-@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
-def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
-    """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = get_crawl_by_ref(crawl_id)
-    payload = data.dict(exclude_unset=True)
-    update_fields = ["modified_at"]
-
-    action = payload.pop("action", None)
-    if action:
-        if action == "pause":
-            crawl.pause()
-            return crawl
-        if action in ("resume", "unpause"):
-            crawl.resume()
-            return crawl
-        if action == "cancel":
-            crawl.cancel()
-            return crawl
-        raise HttpError(400, f"Invalid action: {action}")
-
-    tags = payload.pop("tags", None)
-    tags_str = payload.pop("tags_str", None)
-    if tags is not None or tags_str is not None:
-        crawl.tags_str = ",".join(normalize_tag_list(tags, tags_str or ""))
-        update_fields.append("tags_str")
-
-    if "status" in payload:
-        if payload["status"] not in Crawl.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        if payload["status"] == Crawl.StatusChoices.SEALED:
-            crawl.cancel()
-            return crawl
-        crawl.status = payload["status"]
-        update_fields.append("status")
-
-    if "retry_at" in payload:
-        crawl.retry_at = payload["retry_at"]
-        update_fields.append("retry_at")
-
-    crawl.save(update_fields=update_fields)
-    return crawl
-
-
-@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
-def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = get_crawl_by_ref(crawl_id)
-    crawl_id_str = str(crawl.id)
-    crawl.cancel()
-
-    from archivebox.crawls.locks import crawl_lifecycle_lock
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        crawl = get_crawl_by_ref(crawl_id_str)
-        snapshot_count = crawl.snapshot_set.count()
-        deleted_count, _ = crawl.delete()
-    return {
-        "success": True,
-        "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
-        "deleted_snapshots": snapshot_count,
-    }
@@ -1,161 +0,0 @@
-__package__ = "archivebox.api"
-
-from uuid import UUID
-from typing import Annotated
-from datetime import datetime
-
-from django.http import HttpRequest
-
-from ninja import FilterLookup, FilterSchema, Query, Router, Schema
-from ninja.pagination import paginate
-
-from archivebox.api.v1_core import CustomPagination
-
-
-router = Router(tags=["Machine and Dependencies"])
-
-
-# ============================================================================
-# Machine Schemas
-# ============================================================================
-
-
-class MachineSchema(Schema):
-    """Schema for Machine model."""
-
-    TYPE: str = "machine.Machine"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    guid: str
-    hostname: str
-    hw_in_docker: bool
-    hw_in_vm: bool
-    hw_manufacturer: str
-    hw_product: str
-    hw_uuid: str
-    os_arch: str
-    os_family: str
-    os_platform: str
-    os_release: str
-    os_kernel: str
-    stats: dict
-    num_uses_succeeded: int
-    num_uses_failed: int
-
-
-class MachineFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup("id__startswith")] = None
-    hostname: Annotated[str | None, FilterLookup("hostname__icontains")] = None
-    os_platform: Annotated[str | None, FilterLookup("os_platform__icontains")] = None
-    os_arch: Annotated[str | None, FilterLookup("os_arch")] = None
-    hw_in_docker: Annotated[bool | None, FilterLookup("hw_in_docker")] = None
-    hw_in_vm: Annotated[bool | None, FilterLookup("hw_in_vm")] = None
-    bin_providers: Annotated[str | None, FilterLookup("bin_providers__icontains")] = None
-
-
-# ============================================================================
-# Binary Schemas
-# ============================================================================
-
-
-class BinarySchema(Schema):
-    """Schema for Binary model."""
-
-    TYPE: str = "machine.Binary"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    machine_id: UUID
-    machine_hostname: str
-    name: str
-    binproviders: str
-    binprovider: str
-    abspath: str
-    version: str
-    sha256: str
-    status: str
-    is_valid: bool
-    num_uses_succeeded: int
-    num_uses_failed: int
-
-    @staticmethod
-    def resolve_machine_hostname(obj) -> str:
-        return obj.machine.hostname
-
-    @staticmethod
-    def resolve_is_valid(obj) -> bool:
-        return obj.is_valid
-
-
-class BinaryFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup("id__startswith")] = None
-    name: Annotated[str | None, FilterLookup("name__icontains")] = None
-    binprovider: Annotated[str | None, FilterLookup("binprovider")] = None
-    status: Annotated[str | None, FilterLookup("status")] = None
-    machine_id: Annotated[str | None, FilterLookup("machine_id__startswith")] = None
-    version: Annotated[str | None, FilterLookup("version__icontains")] = None
-
-
-# ============================================================================
-# Machine Endpoints
-# ============================================================================
-
-
-@router.get("/machines", response=list[MachineSchema], url_name="get_machines")
-@paginate(CustomPagination)
-def get_machines(request: HttpRequest, filters: Query[MachineFilterSchema]):
-    """List all machines."""
-    from archivebox.machine.models import Machine
-
-    return filters.filter(Machine.objects.all()).distinct()
-
-
-@router.get("/machine/current", response=MachineSchema, url_name="get_current_machine")
-def get_current_machine(request: HttpRequest):
-    """Get the current machine."""
-    from archivebox.machine.models import Machine
-
-    return Machine.current()
-
-
-@router.get("/machine/{machine_id}", response=MachineSchema, url_name="get_machine")
-def get_machine(request: HttpRequest, machine_id: str):
-    """Get a specific machine by ID."""
-    from archivebox.machine.models import Machine
-    from django.db.models import Q
-
-    return Machine.objects.get(Q(id__startswith=machine_id) | Q(hostname__iexact=machine_id))
-
-
-# ============================================================================
-
-
-# ============================================================================
-# Binary Endpoints
-# ============================================================================
-
-
-@router.get("/binaries", response=list[BinarySchema], url_name="get_binaries")
-@paginate(CustomPagination)
-def get_binaries(request: HttpRequest, filters: Query[BinaryFilterSchema]):
-    """List all binaries."""
-    from archivebox.machine.models import Binary
-
-    return filters.filter(Binary.objects.all().select_related("machine")).distinct()
-
-
-@router.get("/binary/{binary_id}", response=BinarySchema, url_name="get_binary")
-def get_binary(request: HttpRequest, binary_id: str):
-    """Get a specific binary by ID."""
-    from archivebox.machine.models import Binary
-
-    return Binary.objects.select_related("machine").get(id__startswith=binary_id)
-
-
-@router.get("/binary/by-name/{name}", response=list[BinarySchema], url_name="get_binaries_by_name")
-def get_binaries_by_name(request: HttpRequest, name: str):
-    """Get all binaries with the given name."""
-    from archivebox.machine.models import Binary
-
-    return list(Binary.objects.filter(name__iexact=name).select_related("machine"))
@@ -1,168 +0,0 @@
-__package__ = "archivebox.api"
-
-import json
-from datetime import datetime
-from typing import Any
-from uuid import UUID
-
-from django.db.models import Q
-from django.http import HttpRequest
-from ninja import Router, Schema
-from ninja.pagination import paginate
-from pydantic import Field
-
-from archivebox.api.v1_core import CustomPagination
-from archivebox.personas.importers import validate_persona_name
-from archivebox.personas.models import Persona
-
-
-router = Router(tags=["Personas"])
-
-
-class PersonaBrowserSettingsSchema(Schema):
-    user_agent: str = ""
-    viewport_size: str = ""
-    viewport_device_scale_factor: float | None = None
-    language: str = ""
-    timezone: str = ""
-    geolocation: dict[str, Any] | None = None
-
-
-class PersonaSyncSchema(Schema):
-    extension_persona_id: str
-    name: str
-    settings: PersonaBrowserSettingsSchema = Field(default_factory=PersonaBrowserSettingsSchema)
-    cookies_txt: str = ""
-    auth_json: dict[str, Any] = Field(default_factory=dict)
-
-
-class PersonaSchema(Schema):
-    TYPE: str = "personas.models.Persona"
-    id: UUID
-    name: str
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-    config: dict[str, Any] | None
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by.pk)
-
-    @staticmethod
-    def resolve_created_by_username(obj) -> str:
-        return obj.created_by.username
-
-    @staticmethod
-    def resolve_config(obj):
-        # Redact credential values so REST responses don't leak the raw
-        # token/secret/api-key the operator stored in Persona.config.
-        from archivebox.config.common import redact_sensitive_config
-
-        return redact_sensitive_config(obj.config)
-
-
-class PersonaSyncResponseSchema(Schema):
-    success: bool
-    created: bool
-    persona: PersonaSchema
-    cookies_file_written: bool
-    auth_file_written: bool
-
-
-def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrowserSettingsSchema) -> dict[str, Any]:
-    config: dict[str, Any] = {
-        "BROWSER_EXTENSION_PERSONA_ID": extension_persona_id,
-        "BROWSER_EXTENSION_SYNCED_AT": datetime.utcnow().isoformat() + "Z",
-    }
-
-    if settings.user_agent:
-        config.update(
-            {
-                "USER_AGENT": settings.user_agent,
-                "CHROME_USER_AGENT": settings.user_agent,
-                "WGET_USER_AGENT": settings.user_agent,
-                "CURL_USER_AGENT": settings.user_agent,
-            },
-        )
-    if settings.viewport_size:
-        config.update(
-            {
-                "RESOLUTION": settings.viewport_size,
-                "CHROME_RESOLUTION": settings.viewport_size,
-            },
-        )
-    if settings.viewport_device_scale_factor is not None:
-        config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
-    if settings.language:
-        config["BROWSER_LANGUAGE"] = settings.language
-    if settings.timezone:
-        config["BROWSER_TIMEZONE"] = settings.timezone
-    if settings.geolocation:
-        config["BROWSER_GEOLOCATION"] = settings.geolocation
-
-    return config
-
-
-def find_persona(extension_persona_id: str, name: str) -> Persona | None:
-    return (
-        Persona.objects.filter(
-            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
-        )
-        .order_by("created_at")
-        .first()
-    )
-
-
-@router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
-@paginate(CustomPagination)
-def get_personas(request: HttpRequest):
-    """List personas available on this ArchiveBox server."""
-    return Persona.objects.all().order_by("name")
-
-
-@router.post("/sync", response=PersonaSyncResponseSchema, url_name="sync_persona")
-def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
-    """
-    Create or update a Persona from a browser extension profile export.
-
-    The extension sends browser settings plus portable auth artifacts. The server
-    keeps browser override settings in Persona.config and writes cookies.txt /
-    auth.json into the persona directory for extractors to consume.
-    """
-    name = payload.name.strip()
-    is_valid, error_message = validate_persona_name(name)
-    if not is_valid:
-        raise ValueError(error_message)
-
-    persona = find_persona(payload.extension_persona_id, name)
-    created = persona is None
-    if persona is None:
-        persona = Persona(name=name)
-        if request.user.is_authenticated:
-            persona.created_by = request.user
-
-    persona.config = {
-        **(persona.config or {}),
-        **browser_settings_to_config(payload.extension_persona_id, payload.settings),
-    }
-    persona.save()
-    persona.ensure_dirs()
-
-    cookies_written = False
-    if payload.cookies_txt.strip():
-        (persona.path / "cookies.txt").write_text(payload.cookies_txt)
-        cookies_written = True
-
-    auth_written = False
-    if payload.auth_json:
-        (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
-        auth_written = True
-
-    return {
-        "success": True,
-        "created": created,
-        "persona": persona,
-        "cookies_file_written": cookies_written,
-        "auth_file_written": auth_written,
-    }
@@ -1,32 +0,0 @@
-__package__ = "archivebox.api"
-
-from typing import Any
-from collections.abc import Callable
-import logging
-
-from django.db import transaction
-from signal_webhooks.handlers import sync_task_handler
-
-
-logger = logging.getLogger(__name__)
-
-
-def warning_error_handler(hook: Any, error: Exception | None) -> None:
-    if error is not None:
-        logger.warning("Outbound webhook %r failed: %s", hook.name, error)
-        return
-
-    logger.warning("Outbound webhook %r returned a non-success response.", hook.name)
-
-
-def transaction_on_commit_task_handler(hook: Callable[..., None], **kwargs: Any) -> None:
-    def run_webhook() -> None:
-        try:
-            sync_task_handler(hook, **kwargs)
-        except Exception:
-            logger.warning("Outbound webhook failed after transaction commit.", exc_info=True)
-
-    try:
-        transaction.on_commit(run_webhook)
-    except Exception:
-        logger.warning("Could not schedule outbound webhook after transaction commit.", exc_info=True)
@@ -1 +0,0 @@
-__package__ = "archivebox.base_models"
@@ -1,891 +0,0 @@
-"""Base admin classes for models using UUIDv7."""
-
-__package__ = "archivebox.base_models"
-
-import json
-import uuid
-from collections.abc import Mapping
-from typing import NotRequired, TypedDict, cast
-
-from django import forms
-from django.contrib import admin
-from django.db import models
-from django.forms.renderers import BaseRenderer
-from django.http import HttpRequest, QueryDict
-from django.urls import path, register_converter
-from django.utils.safestring import SafeString, mark_safe
-from django_object_actions import DjangoObjectActions
-
-
-class HexUUIDConverter:
-    """URL path converter that canonicalizes UUIDs to their 32-char hex form.
-
-    Accepts both the hyphenated (``aaaaaaaa-bbbb-...``) and bare-hex
-    (``aaaaaaaabbbb...``) UUID strings on the way in (Django's UUIDField
-    parses either), but ``to_url`` always emits the bare-hex form. This is
-    what makes ``reverse("admin:app_model_change", args=[obj.pk])`` produce
-    ``/admin/app/model/06a1a8facb0d.../change/`` instead of the default
-    hyphenated rendering — admin links throughout the app reverse through
-    this converter once ``BaseModelAdmin.get_urls`` swaps in
-    ``<hexuuid:object_id>`` below.
-    """
-
-    regex = r"[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
-
-    def to_python(self, value: str) -> str:
-        # Strip hyphens but stay as a string — Django admin treats the
-        # captured object_id as a string and calls ``model._meta.pk.to_python``
-        # itself, so we don't want to short-circuit that.
-        return value.replace("-", "")
-
-    def to_url(self, value) -> str:
-        if isinstance(value, uuid.UUID):
-            return value.hex
-        return str(value).replace("-", "")
-
-
-register_converter(HexUUIDConverter, "hexuuid")
-
-
-class ConfigOption(TypedDict):
-    plugin: str
-    type: str | list[str]
-    default: object
-    description: str
-    enum: NotRequired[list[object]]
-    pattern: NotRequired[str]
-    minimum: NotRequired[int | float]
-    maximum: NotRequired[int | float]
-
-
-class KeyValueWidget(forms.Widget):
-    """
-    A widget that renders JSON dict as editable key-value input fields
-    with + and - buttons to add/remove rows.
-    Includes autocomplete for available config keys from the plugin system.
-    """
-
-    template_name = ""  # We render manually
-
-    class Media:
-        css = {
-            "all": [],
-        }
-        js = []
-
-    def _get_config_options(self) -> dict[str, ConfigOption]:
-        """Get available config options from plugins."""
-        try:
-            from archivebox.config.common import config_field_metadata
-
-            options: dict[str, ConfigOption] = {}
-            for key, metadata in config_field_metadata().items():
-                option_type = metadata.get("type", "string")
-                option: ConfigOption = {
-                    "plugin": str(metadata.get("plugin", "archivebox")),
-                    "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
-                    "default": metadata.get("default", ""),
-                    "description": str(metadata.get("description", "")),
-                }
-                schema = metadata.get("schema")
-                if isinstance(schema, Mapping):
-                    for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in schema:
-                            option[schema_key] = schema[schema_key]
-                options[key] = option
-            return options
-        except Exception:
-            return {}
-
-    def _parse_value(self, value: object) -> dict[str, object]:
-        # Parse JSON value to dict
-        if value is None:
-            return {}
-        if isinstance(value, str):
-            try:
-                parsed = json.loads(value) if value else {}
-            except json.JSONDecodeError:
-                return {}
-            return parsed if isinstance(parsed, dict) else {}
-        if isinstance(value, Mapping):
-            return {str(key): item for key, item in value.items()}
-        return {}
-
-    def render(
-        self,
-        name: str,
-        value: object,
-        attrs: Mapping[str, str] | None = None,
-        renderer: BaseRenderer | None = None,
-    ) -> SafeString:
-        data = self._parse_value(value)
-
-        widget_id = attrs.get("id", name) if attrs else name
-        config_options = self._get_config_options()
-
-        # Build datalist options
-        datalist_options = "\n".join(
-            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
-            for key, opt in sorted(config_options.items())
-        )
-
-        # Build config metadata as JSON for JS
-        config_meta_json = json.dumps(config_options)
-
-        html = f'''
-        <div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
-            <datalist id="{widget_id}_keys">
-                {datalist_options}
-            </datalist>
-            <div id="{widget_id}_rows" class="key-value-rows">
-        '''
-
-        # Render existing key-value pairs
-        for key, val in data.items():
-            val_str = json.dumps(val) if not isinstance(val, str) else val
-            html += self._render_row(widget_id, key, val_str)
-
-        # Always add one empty row for new entries
-        html += self._render_row(widget_id, "", "")
-
-        html += f'''
-            </div>
-            <div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
-                <button type="button" onclick="addKeyValueRow_{widget_id}()"
-                        style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
-                    + Add Row
-                </button>
-            </div>
-            <input type="hidden" name="{name}" id="{widget_id}" value="">
-            <script>
-                (function() {{
-                    var configMeta_{widget_id} = {config_meta_json};
-                    var rowCounter_{widget_id} = 0;
-
-                    function stringifyValue_{widget_id}(value) {{
-                        return typeof value === 'string' ? value : JSON.stringify(value);
-                    }}
-
-                    function getTypes_{widget_id}(meta) {{
-                        if (!meta || meta.type === undefined || meta.type === null) {{
-                            return [];
-                        }}
-                        return Array.isArray(meta.type) ? meta.type : [meta.type];
-                    }}
-
-                    function getMetaForKey_{widget_id}(key) {{
-                        if (!key) {{
-                            return null;
-                        }}
-
-                        var explicitMeta = configMeta_{widget_id}[key];
-                        if (explicitMeta) {{
-                            return Object.assign({{ key: key }}, explicitMeta);
-                        }}
-
-                        if (key.endsWith('_BINARY')) {{
-                            return {{
-                                key: key,
-                                plugin: 'custom',
-                                type: 'string',
-                                default: '',
-                                description: 'Path to binary executable',
-                            }};
-                        }}
-
-                        if (isRegexConfigKey_{widget_id}(key)) {{
-                            return {{
-                                key: key,
-                                plugin: 'custom',
-                                type: 'string',
-                                default: '',
-                                description: 'Regex pattern list',
-                            }};
-                        }}
-
-                        return null;
-                    }}
-
-                    function describeMeta_{widget_id}(meta) {{
-                        if (!meta) {{
-                            return '';
-                        }}
-
-                        var details = '';
-                        if (Array.isArray(meta.enum) && meta.enum.length) {{
-                            details = 'Allowed: ' + meta.enum.map(stringifyValue_{widget_id}).join(', ');
-                        }} else {{
-                            var types = getTypes_{widget_id}(meta);
-                            if (types.length) {{
-                                details = 'Expected: ' + types.join(' or ');
-                            }}
-                        }}
-
-                        if (meta.minimum !== undefined || meta.maximum !== undefined) {{
-                            var bounds = [];
-                            if (meta.minimum !== undefined) bounds.push('min ' + meta.minimum);
-                            if (meta.maximum !== undefined) bounds.push('max ' + meta.maximum);
-                            details += (details ? ' ' : '') + '(' + bounds.join(', ') + ')';
-                        }}
-
-                        return [meta.description || '', details].filter(Boolean).join(' ');
-                    }}
-
-                    function getExampleInput_{widget_id}(key, meta) {{
-                        var types = getTypes_{widget_id}(meta);
-                        if (key.endsWith('_BINARY')) {{
-                            return 'Example: wget or /usr/bin/wget';
-                        }}
-                        if (key.endsWith('_ARGS_EXTRA') || key.endsWith('_ARGS')) {{
-                            return 'Example: ["--extra-arg"]';
-                        }}
-                        if (types.includes('array')) {{
-                            return 'Example: ["value"]';
-                        }}
-                        if (types.includes('object')) {{
-                            return 'Example: {{"key": "value"}}';
-                        }}
-                        return '';
-                    }}
-
-                    function isRegexConfigKey_{widget_id}(key) {{
-                        return key === 'URL_ALLOWLIST' ||
-                            key === 'URL_DENYLIST' ||
-                            key.endsWith('_PATTERN') ||
-                            key.includes('REGEX');
-                    }}
-
-                    function isSimpleFilterPattern_{widget_id}(pattern) {{
-                        return /^[\\w.*:-]+$/.test(pattern);
-                    }}
-
-                    function validateRegexPattern_{widget_id}(pattern) {{
-                        if (!pattern || isSimpleFilterPattern_{widget_id}(pattern)) {{
-                            return '';
-                        }}
-
-                        try {{
-                            new RegExp(pattern);
-                        }} catch (error) {{
-                            return error && error.message ? error.message : 'Invalid regex';
-                        }}
-                        return '';
-                    }}
-
-                    function validateRegexConfig_{widget_id}(key, raw, typeName) {{
-                        if (typeName === 'object') {{
-                            var parsed;
-                            try {{
-                                parsed = JSON.parse(raw);
-                            }} catch (error) {{
-                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
-                            }}
-                            if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {{
-                                return {{ ok: false, value: parsed, message: 'Must be a JSON object' }};
-                            }}
-                            for (var regexKey in parsed) {{
-                                var objectRegexError = validateRegexPattern_{widget_id}(regexKey);
-                                if (objectRegexError) {{
-                                    return {{ ok: false, value: parsed, message: 'Invalid regex key "' + regexKey + '": ' + objectRegexError }};
-                                }}
-                            }}
-                            return {{ ok: true, value: parsed, message: '' }};
-                        }}
-
-                        var patterns = raw.split(/[\\n,]+/).map(function(pattern) {{
-                            return pattern.trim();
-                        }}).filter(Boolean);
-                        for (var i = 0; i < patterns.length; i++) {{
-                            var regexError = validateRegexPattern_{widget_id}(patterns[i]);
-                            if (regexError) {{
-                                return {{ ok: false, value: raw, message: 'Invalid regex "' + patterns[i] + '": ' + regexError }};
-                            }}
-                        }}
-                        return {{ ok: true, value: raw, message: '' }};
-                    }}
-
-                    function validateBinaryValue_{widget_id}(raw) {{
-                        if (!raw) {{
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        if (/['"`]/.test(raw)) {{
-                            return {{ ok: false, value: raw, message: 'Binary paths cannot contain quotes' }};
-                        }}
-
-                        if (/[;&|<>$(){{}}\\[\\]!]/.test(raw)) {{
-                            return {{ ok: false, value: raw, message: 'Binary paths can only be a binary name or absolute path' }};
-                        }}
-
-                        if (raw.startsWith('/')) {{
-                            if (/^[A-Za-z0-9_./+\\- ]+$/.test(raw)) {{
-                                return {{ ok: true, value: raw, message: '' }};
-                            }}
-                            return {{ ok: false, value: raw, message: 'Absolute paths may only contain path-safe characters' }};
-                        }}
-
-                        if (/^[A-Za-z0-9_.+-]+$/.test(raw)) {{
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        return {{ ok: false, value: raw, message: 'Enter a binary name like wget or an absolute path like /usr/bin/wget' }};
-                    }}
-
-                    function parseValue_{widget_id}(raw) {{
-                        try {{
-                            if (raw === 'true') return true;
-                            if (raw === 'false') return false;
-                            if (raw === 'null') return null;
-                            if (raw !== '' && !isNaN(raw)) return Number(raw);
-                            if ((raw.startsWith('{{') && raw.endsWith('}}')) ||
-                                (raw.startsWith('[') && raw.endsWith(']')) ||
-                                (raw.startsWith('"') && raw.endsWith('"'))) {{
-                                return JSON.parse(raw);
-                            }}
-                        }} catch (error) {{
-                            return raw;
-                        }}
-                        return raw;
-                    }}
-
-                    function sameValue_{widget_id}(left, right) {{
-                        return left === right || JSON.stringify(left) === JSON.stringify(right);
-                    }}
-
-                    function parseTypedValue_{widget_id}(raw, typeName, meta) {{
-                        var numberValue;
-                        var parsed;
-
-                        if (typeName && meta && meta.key && isRegexConfigKey_{widget_id}(meta.key)) {{
-                            return validateRegexConfig_{widget_id}(meta.key, raw, typeName);
-                        }}
-
-                        if (typeName === 'string' && meta && meta.key && meta.key.endsWith('_BINARY')) {{
-                            return validateBinaryValue_{widget_id}(raw);
-                        }}
-
-                        if (typeName === 'string') {{
-                            if (meta.pattern) {{
-                                try {{
-                                    if (!(new RegExp(meta.pattern)).test(raw)) {{
-                                        return {{ ok: false, value: raw, message: 'Must match pattern ' + meta.pattern }};
-                                    }}
-                                }} catch (error) {{}}
-                            }}
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        if (typeName === 'integer') {{
-                            if (!/^-?\\d+$/.test(raw)) {{
-                                return {{ ok: false, value: raw, message: 'Must be an integer' }};
-                            }}
-                            numberValue = Number(raw);
-                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
-                            }}
-                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
-                            }}
-                            return {{ ok: true, value: numberValue, message: '' }};
-                        }}
-
-                        if (typeName === 'number') {{
-                            if (raw === '' || isNaN(raw)) {{
-                                return {{ ok: false, value: raw, message: 'Must be a number' }};
-                            }}
-                            numberValue = Number(raw);
-                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
-                            }}
-                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
-                            }}
-                            return {{ ok: true, value: numberValue, message: '' }};
-                        }}
-
-                        if (typeName === 'boolean') {{
-                            var lowered = raw.toLowerCase();
-                            if (lowered === 'true' || raw === '1') return {{ ok: true, value: true, message: '' }};
-                            if (lowered === 'false' || raw === '0') return {{ ok: true, value: false, message: '' }};
-                            return {{ ok: false, value: raw, message: 'Must be true or false' }};
-                        }}
-
-                        if (typeName === 'null') {{
-                            return raw === 'null'
-                                ? {{ ok: true, value: null, message: '' }}
-                                : {{ ok: false, value: raw, message: 'Must be null' }};
-                        }}
-
-                        if (typeName === 'array' || typeName === 'object') {{
-                            try {{
-                                parsed = JSON.parse(raw);
-                            }} catch (error) {{
-                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
-                            }}
-
-                            if (typeName === 'array' && Array.isArray(parsed)) {{
-                                return {{ ok: true, value: parsed, message: '' }};
-                            }}
-                            if (typeName === 'object' && parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {{
-                                return {{ ok: true, value: parsed, message: '' }};
-                            }}
-
-                            return {{
-                                ok: false,
-                                value: parsed,
-                                message: typeName === 'array' ? 'Must be a JSON array' : 'Must be a JSON object',
-                            }};
-                        }}
-
-                        return {{ ok: true, value: parseValue_{widget_id}(raw), message: '' }};
-                    }}
-
-                    function validateValueAgainstMeta_{widget_id}(raw, meta) {{
-                        if (!meta || raw === '') {{
-                            return {{ state: 'neutral', value: raw, message: '' }};
-                        }}
-
-                        var enumValues = Array.isArray(meta.enum) ? meta.enum : [];
-                        var types = getTypes_{widget_id}(meta);
-                        if (!types.length) {{
-                            types = ['string'];
-                        }}
-
-                        var error = 'Invalid value';
-                        for (var i = 0; i < types.length; i++) {{
-                            var candidate = parseTypedValue_{widget_id}(raw, types[i], meta);
-                            if (!candidate.ok) {{
-                                error = candidate.message || error;
-                                continue;
-                            }}
-                            if (enumValues.length && !enumValues.some(function(enumValue) {{
-                                return sameValue_{widget_id}(enumValue, candidate.value) || stringifyValue_{widget_id}(enumValue) === raw;
-                            }})) {{
-                                error = 'Must be one of: ' + enumValues.map(stringifyValue_{widget_id}).join(', ');
-                                continue;
-                            }}
-                            return {{ state: 'valid', value: candidate.value, message: '' }};
-                        }}
-
-                        return {{ state: 'invalid', value: raw, message: error }};
-                    }}
-
-                    function ensureRowId_{widget_id}(row) {{
-                        if (!row.dataset.rowId) {{
-                            row.dataset.rowId = String(rowCounter_{widget_id}++);
-                        }}
-                        return row.dataset.rowId;
-                    }}
-
-                    function setRowHelp_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var help = row.querySelector('.kv-help');
-                        if (!keyInput || !help) {{
-                            return;
-                        }}
-
-                        var key = keyInput.value.trim();
-                        if (!key) {{
-                            help.textContent = '';
-                            return;
-                        }}
-
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (meta) {{
-                            var extra = isRegexConfigKey_{widget_id}(key)
-                                ? ((meta.type === 'object' || (Array.isArray(meta.type) && meta.type.includes('object')))
-                                    ? ' Expected: JSON object with regex keys.'
-                                    : ' Expected: valid regex.')
-                                : '';
-                            var example = getExampleInput_{widget_id}(key, meta);
-                            help.textContent = [describeMeta_{widget_id}(meta) + extra, example].filter(Boolean).join(' ');
-                        }} else {{
-                            help.textContent = 'Custom key';
-                        }}
-                    }}
-
-                    function configureValueInput_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var valueInput = row.querySelector('.kv-value');
-                        var datalist = row.querySelector('.kv-value-options');
-                        if (!keyInput || !valueInput || !datalist) {{
-                            return;
-                        }}
-
-                        var rowId = ensureRowId_{widget_id}(row);
-                        datalist.id = '{widget_id}_value_options_' + rowId;
-
-                        var meta = getMetaForKey_{widget_id}(keyInput.value.trim());
-                        var enumValues = Array.isArray(meta && meta.enum) ? meta.enum : [];
-                        var types = getTypes_{widget_id}(meta);
-                        if (!enumValues.length && types.includes('boolean')) {{
-                            enumValues = ['True', 'False'];
-                        }}
-                        if (enumValues.length) {{
-                            datalist.innerHTML = enumValues.map(function(enumValue) {{
-                                return '<option value="' + stringifyValue_{widget_id}(enumValue).replace(/"/g, '&quot;') + '"></option>';
-                            }}).join('');
-                            valueInput.setAttribute('list', datalist.id);
-                        }} else {{
-                            datalist.innerHTML = '';
-                            valueInput.removeAttribute('list');
-                        }}
-                    }}
-
-                    function setValueValidationState_{widget_id}(input, state, message) {{
-                        if (!input) {{
-                            return;
-                        }}
-
-                        if (state === 'valid') {{
-                            input.style.borderColor = '#2da44e';
-                            input.style.boxShadow = '0 0 0 1px rgba(45, 164, 78, 0.18)';
-                            input.style.backgroundColor = '#f6ffed';
-                        }} else if (state === 'invalid') {{
-                            input.style.borderColor = '#cf222e';
-                            input.style.boxShadow = '0 0 0 1px rgba(207, 34, 46, 0.18)';
-                            input.style.backgroundColor = '#fff8f8';
-                        }} else {{
-                            input.style.borderColor = '#ccc';
-                            input.style.boxShadow = 'none';
-                            input.style.backgroundColor = '';
-                        }}
-                        input.title = message || '';
-                    }}
-
-                    function applyValueValidation_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var valueInput = row.querySelector('.kv-value');
-                        if (!keyInput || !valueInput) {{
-                            return;
-                        }}
-
-                        var key = keyInput.value.trim();
-                        if (!key) {{
-                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
-                            return;
-                        }}
-
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (!meta) {{
-                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
-                            return;
-                        }}
-
-                        var validation = validateValueAgainstMeta_{widget_id}(valueInput.value.trim(), meta);
-                        setValueValidationState_{widget_id}(valueInput, validation.state, validation.message);
-                    }}
-
-                    function coerceValueForStorage_{widget_id}(key, raw) {{
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (!meta) {{
-                            return parseValue_{widget_id}(raw);
-                        }}
-
-                        var validation = validateValueAgainstMeta_{widget_id}(raw, meta);
-                        return validation.state === 'valid' ? validation.value : raw;
-                    }}
-
-                    function initializeRows_{widget_id}() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
-                            ensureRowId_{widget_id}(row);
-                            configureValueInput_{widget_id}(row);
-                            setRowHelp_{widget_id}(row);
-                            applyValueValidation_{widget_id}(row);
-                        }});
-                    }}
-
-                    function updateHiddenField_{widget_id}() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        var rows = container.querySelectorAll('.key-value-row');
-                        var result = {{}};
-                        rows.forEach(function(row) {{
-                            var keyInput = row.querySelector('.kv-key');
-                            var valInput = row.querySelector('.kv-value');
-                            if (keyInput && valInput && keyInput.value.trim()) {{
-                                var key = keyInput.value.trim();
-                                var val = valInput.value.trim();
-                                result[key] = coerceValueForStorage_{widget_id}(key, val);
-                            }}
-                        }});
-                        document.getElementById('{widget_id}').value = JSON.stringify(result);
-                    }}
-
-                    window.addKeyValueRow_{widget_id} = function() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        var newRow = document.createElement('div');
-                        newRow.className = 'key-value-row';
-                        newRow.style.cssText = 'margin-bottom: 6px;';
-                        newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
-                            '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
-                            'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
-                            '<input type="text" class="kv-value" placeholder="value" ' +
-                            'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
-                            '<datalist class="kv-value-options"></datalist>' +
-                            '<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
-                            'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>' +
-                            '</div>' +
-                            '<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>';
-                        container.appendChild(newRow);
-                        ensureRowId_{widget_id}(newRow);
-                        configureValueInput_{widget_id}(newRow);
-                        setRowHelp_{widget_id}(newRow);
-                        applyValueValidation_{widget_id}(newRow);
-                        updateHiddenField_{widget_id}();
-                        newRow.querySelector('.kv-key').focus();
-                    }};
-
-                    window.removeKeyValueRow_{widget_id} = function(btn) {{
-                        var row = btn.closest('.key-value-row');
-                        row.remove();
-                        updateHiddenField_{widget_id}();
-                    }};
-
-                    window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
-
-                    function focusConfigKeyFromHash_{widget_id}() {{
-                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
-                        // to (or creates) the matching row in this editor. Used by
-                        // the in-banner "pin via admin" link and the
-                        // ``→ Edit <KEY> in Machine.config`` shortcut on the live
-                        // config detail page.
-                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
-                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
-                            return;
-                        }}
-                        var container = document.getElementById('{widget_id}_rows');
-                        if (!container) {{
-                            return;
-                        }}
-                        var match = null;
-                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
-                            if (match) {{ return; }}
-                            var keyInput = row.querySelector('.kv-key');
-                            if (keyInput && keyInput.value.trim() === hash) {{
-                                match = row;
-                            }}
-                        }});
-                        if (!match) {{
-                            // No existing row for this key — prepopulate one with the
-                            // key filled in but value left blank so the operator just
-                            // types/pastes the value and hits save.
-                            window.addKeyValueRow_{widget_id}();
-                            var rows = container.querySelectorAll('.key-value-row');
-                            match = rows[rows.length - 1];
-                            var keyInput = match.querySelector('.kv-key');
-                            if (keyInput) {{
-                                keyInput.value = hash;
-                                keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
-                            }}
-                        }}
-                        if (!match) {{
-                            return;
-                        }}
-                        match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
-                        var prevOutline = match.style.outline;
-                        match.style.outline = '2px solid #f59e0b';
-                        match.style.outlineOffset = '2px';
-                        match.style.transition = 'outline 1.2s ease-out';
-                        setTimeout(function() {{
-                            match.style.outline = prevOutline || 'none';
-                        }}, 1400);
-                        var valueInput = match.querySelector('.kv-value');
-                        if (valueInput) {{
-                            valueInput.focus();
-                            try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
-                        }}
-                    }}
-
-                    // Initialize on load
-                    document.addEventListener('DOMContentLoaded', function() {{
-                        initializeRows_{widget_id}();
-                        updateHiddenField_{widget_id}();
-                        focusConfigKeyFromHash_{widget_id}();
-                    }});
-                    // Also run immediately in case DOM is already ready
-                    if (document.readyState !== 'loading') {{
-                        initializeRows_{widget_id}();
-                        updateHiddenField_{widget_id}();
-                        focusConfigKeyFromHash_{widget_id}();
-                    }}
-
-                    window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
-
-                    // Update on any input change
-                    var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
-
-                    rowsEl_{widget_id}.addEventListener('input', function(event) {{
-                        var row = event.target.closest('.key-value-row');
-                        if (!row) {{
-                            return;
-                        }}
-
-                        if (event.target.classList.contains('kv-key')) {{
-                            configureValueInput_{widget_id}(row);
-                            setRowHelp_{widget_id}(row);
-                        }}
-
-                        if (event.target.classList.contains('kv-key') || event.target.classList.contains('kv-value')) {{
-                            applyValueValidation_{widget_id}(row);
-                            updateHiddenField_{widget_id}();
-                        }}
-                    }});
-                }})();
-            </script>
-        </div>
-        '''
-        return mark_safe(html)
-
-    def _render_row(self, widget_id: str, key: str, value: str) -> str:
-        from archivebox.config.common import is_sensitive_config_key
-
-        # Sensitive keys (``*TOKEN*``, ``*SECRET*``, ``*API_KEY*``, ``*APIKEY*``) are
-        # rendered write-only: the input is a password field with a placeholder
-        # showing the value is set, but the raw value is NEVER sent to the browser.
-        # When the user submits the form with the field left blank, the
-        # ``ConfigEditorMixin.save_model`` hook re-merges the previously-saved
-        # value so leaving it untouched is a no-op rather than a destructive clear.
-        is_sensitive = is_sensitive_config_key(key)
-        has_value = bool(value)
-        if is_sensitive:
-            input_type = "password"
-            rendered_value = ""
-            placeholder = (
-                "•••••• (saved — enter new value to replace, clear by deleting row)" if has_value else "value (will be saved write-only)"
-            )
-            extra_attrs = ' autocomplete="off" data-sensitive="1"' + (' data-had-value="1"' if has_value else "")
-        else:
-            input_type = "text"
-            rendered_value = self._escape(value)
-            placeholder = "value"
-            extra_attrs = ""
-        return f'''
-            <div class="key-value-row" style="margin-bottom: 6px;">
-                <div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
-                    <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
-                           style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <input type="{input_type}" class="kv-value" value="{rendered_value}" placeholder="{self._escape(placeholder)}"{extra_attrs}
-                           style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <datalist class="kv-value-options"></datalist>
-                    <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
-                            style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>
-                </div>
-                <div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>
-            </div>
-        '''
-
-    def _escape(self, s: object) -> str:
-        """Escape HTML special chars in attribute values."""
-        if not s:
-            return ""
-        return str(s).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
-
-    def value_from_datadict(
-        self,
-        data: QueryDict | Mapping[str, object],
-        files: object,
-        name: str,
-    ) -> str:
-        value = data.get(name, "{}")
-        return value if isinstance(value, str) else "{}"
-
-
-class ConfigEditorMixin(admin.ModelAdmin):
-    """
-    Mixin for admin classes with a config JSON field.
-
-    Provides a key-value editor widget with autocomplete for available config keys.
-    """
-
-    def formfield_for_dbfield(
-        self,
-        db_field: models.Field,
-        request: HttpRequest,
-        **kwargs: object,
-    ) -> forms.Field | None:
-        """Use KeyValueWidget for the config JSON field."""
-        if db_field.name == "config":
-            kwargs["widget"] = KeyValueWidget()
-        return super().formfield_for_dbfield(db_field, request, **kwargs)
-
-    def save_model(self, request: HttpRequest, obj, form, change):
-        """Preserve write-only redacted credentials on save.
-
-        The KeyValueWidget renders sensitive keys (``*TOKEN*``, ``*SECRET*``,
-        ``*API_KEY*``, ``*APIKEY*``) with an empty value + password input —
-        the real value never leaves the server. On submit, an empty value
-        for a sensitive key that was previously set means "leave untouched",
-        not "clear it." We honor that here by re-merging the stored value
-        before the row is written. Explicitly removing the row in the UI
-        still clears it (the key is gone from the submitted JSON, so there's
-        nothing to merge over).
-        """
-        from archivebox.config.common import is_sensitive_config_key
-
-        if change and obj.pk and obj.config is not None:
-            try:
-                stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
-            except Exception:
-                stored = {}
-            if isinstance(stored, dict):
-                new_config = dict(obj.config or {})
-                for key, new_value in list(new_config.items()):
-                    if not is_sensitive_config_key(key):
-                        continue
-                    if new_value not in (None, "") and new_value != "********":
-                        continue
-                    if key in stored:
-                        new_config[key] = stored[key]
-                obj.config = new_config
-        super().save_model(request, obj, form, change)
-
-
-class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
-    list_display = ("id", "created_at", "created_by")
-    readonly_fields = ("id", "created_at", "modified_at")
-    show_search_mode_selector = False
-
-    def get_default_search_mode(self) -> str:
-        # The shared changelist template always asks every admin for a default
-        # search mode, even when the search-mode toggle is hidden.
-        return "meta"
-
-    def get_form(
-        self,
-        request: HttpRequest,
-        obj: models.Model | None = None,
-        change: bool = False,
-        **kwargs: object,
-    ):
-        form = super().get_form(request, obj, change=change, **kwargs)
-        if "created_by" in form.base_fields:
-            form.base_fields["created_by"].initial = request.user
-        return form
-
-    def get_urls(self):
-        """Swap the per-object admin URLs from ``<path:object_id>`` to
-        ``<hexuuid:object_id>`` so canonical change/delete/history URLs use the
-        32-char hex form. The hyphenated form still resolves because the
-        converter's regex accepts both — Django reverses through ``to_url``
-        which always emits hex, so links in templates / changelists / inline
-        formsets all canonicalize automatically.
-
-        Non-UUID PKs (an ``IntegerField`` PK on some legacy table, for example)
-        won't match the converter's regex and fall back to the default
-        ``<path:object_id>`` patterns we still include after our swap.
-        """
-        info = self.opts.app_label, self.opts.model_name
-        object_routes = [
-            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="%s_%s_history" % info),
-            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="%s_%s_delete" % info),
-            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="%s_%s_change" % info),
-        ]
-        # Append after super().get_urls() so our patterns are the
-        # *last-registered* ones with the canonical admin URL names — Django's
-        # reverse() picks the later registration when names collide, which is
-        # how we make ``reverse("admin:app_model_change", args=[obj.pk])``
-        # emit the hex form. The original ``<path:object_id>`` routes stay in
-        # place as a fallback for non-UUID PKs and for resolving inbound
-        # hyphenated URLs (the ``hexuuid`` regex accepts both forms anyway).
-        return super().get_urls() + object_routes
@@ -1,7 +0,0 @@
-# from django.apps import AppConfig
-
-
-# class BaseModelsConfig(AppConfig):
-#     default_auto_field = 'django.db.models.BigAutoField'
-
-#     name = 'base_models'
@@ -1,303 +0,0 @@
-"""Base models using UUIDv7 for all id fields."""
-
-__package__ = "archivebox.base_models"
-
-import json
-import shutil
-from typing import Any
-
-from archivebox.uuid_compat import CompactUUIDField, uuid7
-from pathlib import Path
-
-from django.db import models
-from django.db.models import F
-from django.db import transaction
-from django.db.models.signals import pre_delete
-from django.utils import timezone
-from django.contrib.auth import get_user_model
-from django.urls import reverse_lazy
-from django.conf import settings
-
-from django_stubs_ext.db.models import TypedModelMeta
-
-from archivebox.config import CONSTANTS
-
-
-def normalize_config_json_values(config: Any) -> Any:
-    if not isinstance(config, dict):
-        return config
-
-    normalized = dict(config)
-    for key, value in list(normalized.items()):
-        if not isinstance(value, str) or len(value) < 2:
-            continue
-        if value[:1] != '"' or value[-1:] != '"':
-            continue
-        try:
-            decoded = json.loads(value)
-        except ValueError:
-            continue
-        if isinstance(decoded, str):
-            normalized[key] = decoded
-    return normalized
-
-
-def get_or_create_system_user_pk(username="system"):
-    User = get_user_model()
-    # If there's exactly one superuser, use that for all system operations
-    if User.objects.filter(is_superuser=True).count() == 1:
-        return User.objects.filter(is_superuser=True).values_list("pk", flat=True)[0]
-    # Otherwise get or create the system user
-    user, _ = User.objects.get_or_create(
-        username=username,
-        defaults={"is_staff": True, "is_superuser": True, "email": "", "password": "!"},
-    )
-    return user.pk
-
-
-class AutoDateTimeField(models.DateTimeField):
-    """DateTimeField that automatically updates on save (legacy compatibility)."""
-
-    def pre_save(self, model_instance, add):
-        if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
-            value = timezone.now()
-            setattr(model_instance, self.attname, value)
-            return value
-        return super().pre_save(model_instance, add)
-
-
-class ModelWithUUID(models.Model):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-    created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL,
-        on_delete=models.CASCADE,
-        default=get_or_create_system_user_pk,
-        null=False,
-        db_index=True,
-    )
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def __str__(self) -> str:
-        return f"[{self.id}] {self.__class__.__name__}"
-
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
-    @property
-    def api_url(self) -> str:
-        return str(reverse_lazy("api-1:get_any", args=[self.id]))
-
-    @property
-    def api_docs_url(self) -> str:
-        return f"/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}"
-
-
-class ModelWithNotes(models.Model):
-    """Mixin for models with a notes field."""
-
-    notes = models.TextField(blank=True, null=False, default="")
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-
-class ModelWithHealthStats(models.Model):
-    """Mixin for models with health tracking fields."""
-
-    num_uses_failed = models.PositiveIntegerField(default=0)
-    num_uses_succeeded = models.PositiveIntegerField(default=0)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
-    @property
-    def health(self) -> int:
-        total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
-        return round((self.num_uses_succeeded / total) * 100)
-
-    def increment_health_stats(self, success: bool):
-        """Atomically increment success or failure counter using F() expression."""
-        field = "num_uses_succeeded" if success else "num_uses_failed"
-        type(self).objects.filter(pk=self.pk).update(
-            **{
-                field: F(field) + 1,
-                "modified_at": timezone.now(),
-            },
-        )
-
-
-class ModelWithConfig(models.Model):
-    """Mixin for models with a JSON config field."""
-
-    config = models.JSONField(default=dict, null=True, blank=True, editable=True)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def save(self, *args, **kwargs):
-        normalized_config = normalize_config_json_values(self.config)
-        if normalized_config != self.config:
-            self.config = normalized_config
-            update_fields = kwargs.get("update_fields")
-            if update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
-        super().save(*args, **kwargs)
-
-
-class ModelWithDeleteAfter(models.Model):
-    delete_after_final_statuses: tuple[str, ...] = ()
-    delete_at = models.DateTimeField(default=None, null=True, blank=True, db_index=True)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def save(self, *args, **kwargs):
-        update_fields = kwargs.get("update_fields")
-        if self.delete_at is None:
-            self.set_delete_at_from_config()
-            if self.delete_at is not None and update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
-        super().save(*args, **kwargs)
-
-    def get_delete_after_config_value(self):
-        from archivebox.config.common import get_config
-
-        return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
-
-    def set_delete_at_from_config(self, config_value=None) -> bool:
-        if self.delete_at is not None:
-            return False
-
-        from archivebox.config.common import parse_delete_after
-
-        duration = parse_delete_after(self.get_delete_after_config_value() if config_value is None else config_value)
-        if duration is None:
-            return False
-
-        self.delete_at = (self.created_at or timezone.now()) + duration
-        return True
-
-    @classmethod
-    def missing_delete_at_candidates(cls):
-        return cls.objects.none()
-
-    @classmethod
-    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
-        if backfill_missing:
-            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
-            for obj in missing_delete_at:
-                if obj.set_delete_at_from_config():
-                    cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
-                        delete_at=obj.delete_at,
-                        modified_at=timezone.now(),
-                    )
-
-        # Keep the expiration sweep anchored on delete_at. Some large tables
-        # have millions of final-status rows but almost no retained rows; adding
-        # the status filter before SQLite has narrowed by delete_at can make the
-        # runner scan the hot status index before it claims new work.
-        due_pks = list(
-            cls.objects.filter(delete_at__isnull=False, delete_at__lte=timezone.now())
-            .order_by("delete_at", "pk")
-            .values_list("pk", flat=True)[:batch_size],
-        )
-        if not due_pks:
-            return 0
-
-        queryset = cls.objects.filter(pk__in=due_pks)
-        if cls.delete_after_final_statuses:
-            queryset = queryset.filter(status__in=cls.delete_after_final_statuses)
-
-        count = 0
-        expired = list(queryset.order_by("delete_at", "pk"))
-        for obj in expired:
-            obj.delete()
-            count += 1
-        return count
-
-
-class ModelWithOutputDir(ModelWithUUID):
-    class Meta(ModelWithUUID.Meta):
-        abstract = True
-
-    _delete_signal_registered = False
-
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        output_dir = Path(self.output_dir)
-        # Avoid holding SQLite write transactions open across slow filesystem work.
-        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
-        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
-
-    @property
-    def output_dir_parent(self) -> str:
-        return f"{self._meta.model_name}s"
-
-    @property
-    def output_dir_name(self) -> str:
-        return str(self.id)
-
-    @property
-    def output_dir_str(self) -> str:
-        return f"{self.output_dir_parent}/{self.output_dir_name}"
-
-    @property
-    def output_dir(self) -> Path:
-        raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
-
-    def output_paths_for_delete(self) -> tuple[Path, ...]:
-        return (Path(self.output_dir),)
-
-    @classmethod
-    def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
-        data_dir = CONSTANTS.DATA_DIR.resolve()
-        safe_paths = []
-        for raw_path in paths:
-            path = Path(raw_path)
-            is_safe = False
-            for candidate in (path.absolute(), path.resolve()):
-                try:
-                    candidate.relative_to(data_dir)
-                    is_safe = True
-                    break
-                except ValueError:
-                    continue
-            if not is_safe:
-                raise ValueError(f"Refusing to delete output path outside DATA_DIR: {path}")
-            safe_paths.append(path)
-        return tuple(safe_paths)
-
-    @classmethod
-    def delete_output_paths(cls, paths) -> None:
-        for path in cls.validate_output_paths_for_delete(paths):
-            if path.is_symlink() or path.is_file():
-                path.unlink(missing_ok=True)
-            elif path.is_dir():
-                shutil.rmtree(path, ignore_errors=True)
-
-    @classmethod
-    def register_delete_signal(cls) -> None:
-        if cls._delete_signal_registered:
-            return
-
-        def schedule_output_dir_cleanup(sender, instance, **kwargs):
-            if not isinstance(instance, ModelWithOutputDir):
-                return
-            paths = instance.validate_output_paths_for_delete(instance.output_paths_for_delete())
-            transaction.on_commit(lambda paths=paths: instance.delete_output_paths(paths))
-
-        pre_delete.connect(
-            schedule_output_dir_cleanup,
-            dispatch_uid="archivebox.output_dir_cleanup_on_delete",
-            weak=False,
-        )
-        cls._delete_signal_registered = True
@@ -1,225 +1,264 @@
-__package__ = "archivebox.cli"
-__command__ = "archivebox"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox'
+
 import os
 import sys
+import argparse
+import threading
+
+from time import sleep
+from collections.abc import Mapping
+
+from rich import print
+
+from typing import Optional, List, IO, Union, Iterable
+from pathlib import Path
+
 from importlib import import_module

-import rich_click as click
-from rich.console import Console
+BUILTIN_LIST = list

-from archivebox.config.version import VERSION
+CLI_DIR = Path(__file__).resolve().parent

-STDERR = Console(stderr=True)
+# rewrite setup -> install for backwards compatibility
+if len(sys.argv) > 1 and sys.argv[1] == 'setup':
+    from rich import print
+    print(':warning: [bold red]DEPRECATED[/bold red] `archivebox setup` is deprecated, use `archivebox install` instead')
+    sys.argv[1] = 'install'
+
+if '--debug' in sys.argv:
+    os.environ['DEBUG'] = 'True'
+    sys.argv.remove('--debug')


-if "--debug" in sys.argv:
-    os.environ["DEBUG"] = "True"
-    sys.argv.remove("--debug")
+# def list_subcommands() -> Dict[str, str]:
+#     """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
+#     COMMANDS = []
+#     for filename in os.listdir(CLI_DIR):
+#         if is_cli_module(filename):
+#             subcommand = filename.replace('archivebox_', '').replace('.py', '')
+#             module = import_module('.archivebox_{}'.format(subcommand), __package__)
+#             assert is_valid_cli_module(module, subcommand)
+#             COMMANDS.append((subcommand, module.main.__doc__))
+#             globals()[subcommand] = module.main
+#     display_order = lambda cmd: (
+#         display_first.index(cmd[0])
+#         if cmd[0] in display_first else
+#         100 + len(cmd[0])
+#     )
+#     return dict(sorted(COMMANDS, key=display_order))

-# Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
-# `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
-# the subcommand executes. `--quick-init` is the old server spelling. Strip either from
-# argv here so each subcommand's own click parser never sees it. Ignored for `help` and
-# `init` themselves.
-if "--init" in sys.argv or "--quick-init" in sys.argv:
-    sys.argv = [arg for arg in sys.argv if arg not in ("--init", "--quick-init")]
-    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+# just define it statically, it's much faster:
+SUBCOMMAND_MODULES = {
+    'help': 'archivebox_help',
+    'version': 'archivebox_version' ,
+    
+    'init': 'archivebox_init',
+    'install': 'archivebox_install',
+    ##############################################
+    'config': 'archivebox_config',
+    'add': 'archivebox_add',
+    'remove': 'archivebox_remove',
+    'update': 'archivebox_update',
+    'list': 'archivebox_list',
+    'status': 'archivebox_status',
+    
+    'schedule': 'archivebox_schedule',
+    'server': 'archivebox_server',
+    'shell': 'archivebox_shell',
+    'manage': 'archivebox_manage',
+
+    # 'oneshot': 'archivebox_oneshot',
+}
+
+# every imported command module must have these properties in order to be valid
+required_attrs = ('__package__', '__command__', 'main')
+
+# basic checks to make sure imported files are valid subcommands
+is_cli_module = lambda fname: fname.startswith('archivebox_') and fname.endswith('.py')
+is_valid_cli_module = lambda module, subcommand: (
+    all(hasattr(module, attr) for attr in required_attrs)
+    and module.__command__.split(' ')[-1] == subcommand
+)
+
+class LazySubcommands(Mapping):
+    def keys(self):
+        return SUBCOMMAND_MODULES.keys()
+    
+    def values(self):
+        return [self[key] for key in self.keys()]
+    
+    def items(self):
+        return [(key, self[key]) for key in self.keys()]
+    
+    def __getitem__(self, key):
+        module = import_module(f'.{SUBCOMMAND_MODULES[key]}', __package__)
+        assert is_valid_cli_module(module, key)
+        return module.main
+    
+    def __iter__(self):
+        return iter(SUBCOMMAND_MODULES.keys())
+    
+    def __len__(self):
+        return len(SUBCOMMAND_MODULES)
+
+CLI_SUBCOMMANDS = LazySubcommands()


-class ArchiveBoxGroup(click.Group):
-    """lazy loading click group for archivebox commands"""
+# these common commands will appear sorted before any others for ease-of-use
+meta_cmds = ('help', 'version')                               # dont require valid data folder at all
+setup_cmds = ('init', 'setup', 'install')                      # require valid data folder, but dont require DB present in it yet
+archive_cmds = ('add', 'remove', 'update', 'list', 'status', 'schedule', 'server', 'shell', 'manage')  # require valid data folder + existing db present
+fake_db = ("oneshot",)                                        # use fake in-memory db

-    meta_commands = {
-        "help": "archivebox.cli.archivebox_help.main",
-        "version": "archivebox.cli.archivebox_version.main",
-        "mcp": "archivebox.cli.archivebox_mcp.main",
-        "oneshot": "archivebox.cli.archivebox_oneshot.main",
-    }
-    setup_commands = {
-        "init": "archivebox.cli.archivebox_init.main",
-        "install": "archivebox.cli.archivebox_install.main",
-    }
-    # Model commands (CRUD operations via subcommands)
-    model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot.main",
-        "archiveresult": "archivebox.cli.archivebox_archiveresult.main",
-        "tag": "archivebox.cli.archivebox_tag.main",
-        "binary": "archivebox.cli.archivebox_binary.main",
-        "process": "archivebox.cli.archivebox_process.main",
-        "machine": "archivebox.cli.archivebox_machine.main",
-        "persona": "archivebox.cli.archivebox_persona.main",
-    }
-    archive_commands = {
-        # High-level commands
-        "add": "archivebox.cli.archivebox_add.main",
-        "extract": "archivebox.cli.archivebox_extract.main",
-        "list": "archivebox.cli.archivebox_list.main",
-        "remove": "archivebox.cli.archivebox_remove.main",
-        "run": "archivebox.cli.archivebox_run.main",
-        "update": "archivebox.cli.archivebox_update.main",
-        "status": "archivebox.cli.archivebox_status.main",
-        "search": "archivebox.cli.archivebox_search.main",
-        "config": "archivebox.cli.archivebox_config.main",
-        "schedule": "archivebox.cli.archivebox_schedule.main",
-        "server": "archivebox.cli.archivebox_server.main",
-        "shell": "archivebox.cli.archivebox_shell.main",
-        "manage": "archivebox.cli.archivebox_manage.main",
-        # Introspection commands
-        "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
-    }
-    all_subcommands = {
-        **meta_commands,
-        **setup_commands,
-        **model_commands,
-        **archive_commands,
-    }
-    renamed_commands = {
-        "setup": "install",
-        "import": "add",
-        "archive": "add",
-    }
-
-    @classmethod
-    def get_canonical_name(cls, cmd_name):
-        return cls.renamed_commands.get(cmd_name, cmd_name)
-
-    @classmethod
-    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
-        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
-        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)
-
-    @classmethod
-    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
-        if not cls._needs_django_for_lazy_import(cmd_name):
-            return
-
-        from django.apps import apps
-
-        if apps.ready:
-            return
-
-        from archivebox.config.django import setup_django
-
-        setup_django()
-
-    def get_command(self, ctx, cmd_name):
-        # handle renamed commands
-        if cmd_name in self.renamed_commands:
-            new_name = self.renamed_commands[cmd_name]
-            STDERR.print(
-                f" [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`",
-            )
-            cmd_name = new_name
-            ctx.invoked_subcommand = cmd_name
-
-        # handle lazy loading of commands
-        if cmd_name in self.all_subcommands:
-            self._setup_django_for_lazy_import(cmd_name)
-            return self._lazy_load(cmd_name)
-
-        # fall-back to using click's default command lookup
-        return super().get_command(ctx, cmd_name)
-
-    @classmethod
-    def _lazy_load(cls, cmd_name_or_path):
-        import_path = cls.all_subcommands.get(cmd_name_or_path)
-        if import_path is None:
-            import_path = cmd_name_or_path
-        modname, funcname = import_path.rsplit(".", 1)
-
-        # print(f'LAZY LOADING {import_path}')
-        mod = import_module(modname)
-        func = vars(mod)[funcname]
-
-        if func.__doc__ is None:
-            raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
-
-        # if not isinstance(cmd, click.BaseCommand):
-        # raise ValueError(f'lazy loading of {import_path} failed - not a click command')
-
-        return func
+display_first = (*meta_cmds, *setup_cmds, *archive_cmds)


-@click.group(cls=ArchiveBoxGroup, invoke_without_command=True)
-@click.option("--help", "-h", is_flag=True, help="Show help")
-@click.version_option(VERSION, "-v", "--version", package_name="archivebox", message="%(version)s")
-@click.pass_context
-def cli(ctx, help=False):
-    """ArchiveBox: The self-hosted internet archive"""
-
-    subcommand = ArchiveBoxGroup.get_canonical_name(ctx.invoked_subcommand)
-
-    # if --help is passed or no subcommand is given, show custom help message
-    if help or ctx.invoked_subcommand is None:
-        ctx.invoke(ctx.command.get_command(ctx, "help"))
-
-    # if the subcommand is in archive_commands or model_commands,
-    # then we need to set up the django environment and check that we're in a valid data folder
-    wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
-    if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
-        # print('SETUP DJANGO AND CHECK DATA FOLDER')
-        try:
-            if subcommand == "server":
-                run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes")
-                if run_in_debug:
-                    os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
-                    if "--reload" in sys.argv:
-                        os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
-
-            from archivebox.config.django import setup_django
-            from archivebox.misc.checks import check_data_folder, check_migrations
-
-            setup_django()
-            if os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1" and subcommand not in ("init", "help"):
-                # Universal `--init` was passed: build/upgrade the data folder before
-                # the regular preflight runs, so it succeeds on a fresh dir and an
-                # out-of-date schema both. Drop the env var afterwards so spawned
-                # subprocesses (supervisord workers, daphne, runner, etc.) inherit
-                # a clean env and don't re-trigger init in every child.
-                from archivebox.cli.archivebox_init import init as archivebox_init
-
-                archivebox_init(quick=True)
-                os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
-            check_data_folder()
-            if subcommand != "update":
-                check_migrations(auto_apply=True)
-        except Exception as e:
-            STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
-            if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
-                raise
+IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread', 'Scheduler')  # threads we dont have to wait for before exiting


-def main(args=None, prog_name=None):
-    # show `docker run archivebox xyz` in help messages if running in docker
-    IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
-    IS_TTY = sys.stdin.isatty()
-    prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
+def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: Iterable[str]=IGNORED_BG_THREADS, timeout: int=60) -> int:
+    """
+    Block until the specified threads exit. e.g. pass thread_names=('default_hook_handler',) to wait for webhooks.
+    Useful for waiting for signal handlers, webhooks, etc. to finish running after a mgmt command completes.
+    """

-    previous_unraisablehook = sys.unraisablehook
+    wait_for_all: bool = thread_names == ()

-    def ignore_shutdown_unraisable(unraisable):
-        if isinstance(unraisable.exc_value, (KeyboardInterrupt, SystemExit)):
-            return
-        previous_unraisablehook(unraisable)
+    thread_matches = lambda thread, ptns: any(ptn in repr(thread) for ptn in ptns)
+
+    should_wait = lambda thread: (
+        not thread_matches(thread, ignore_names)
+        and (wait_for_all or thread_matches(thread, thread_names)))
+
+    for tries in range(timeout):
+        all_threads = [*threading.enumerate()]
+        blocking_threads = [*filter(should_wait, all_threads)]
+        threads_summary = ', '.join(repr(t) for t in blocking_threads)
+        if blocking_threads:
+            sleep(1)
+            if tries == 5:                            # only show stderr message if we need to wait more than 5s
+                print(
+                    f'[…] Waiting up to {timeout}s for background jobs (e.g. webhooks) to finish...',
+                    threads_summary,
+                    file=sys.stderr,
+                )
+        else:
+            return tries
+
+    raise Exception(f'Background threads failed to exit after {tries}s: {threads_summary}')
+
+
+
+def run_subcommand(subcommand: str,
+                   subcommand_args: List[str] | None = None,
+                   stdin: Optional[IO]=None,
+                   pwd: Union[Path, str, None]=None) -> None:
+    """Run a given ArchiveBox subcommand with the given list of args"""
+
+    subcommand_args = subcommand_args or []
+
+    from archivebox.misc.checks import check_migrations
+    from archivebox.config.django import setup_django
+    
+    # print('DATA_DIR is', DATA_DIR)
+    # print('pwd is', os.getcwd())    
+
+    cmd_requires_db = (subcommand in archive_cmds)
+    init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
+
+    check_db = cmd_requires_db and not init_pending
+
+    setup_django(in_memory_db=subcommand in fake_db, check_db=check_db)
+
+    for ignore_pattern in ('help', '-h', '--help', 'version', '--version'):
+        if ignore_pattern in sys.argv[:4]:
+            cmd_requires_db = False
+            break
+    
+    if subcommand in archive_cmds:
+        if cmd_requires_db:
+            check_migrations()
+
+    module = import_module('.archivebox_{}'.format(subcommand), __package__)
+    module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore
+
+    # wait for webhooks, signals, and other background jobs to finish before exit
+    wait_for_bg_threads_to_exit(timeout=60)
+
+
+
+
+
+class NotProvided:
+    def __len__(self):
+        return 0
+    def __bool__(self):
+        return False
+    def __repr__(self):
+        return '<not provided>'
+
+Omitted = Union[None, NotProvided]
+
+OMITTED = NotProvided()
+
+
+def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: str | None=None) -> None:
+    # print('STARTING CLI MAIN ENTRYPOINT')
+    
+    args = sys.argv[1:] if args is OMITTED else args
+    stdin = sys.stdin if stdin is OMITTED else stdin
+
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description='ArchiveBox: The self-hosted internet archive',
+        add_help=False,
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--help', '-h',
+        action='store_true',
+        help=CLI_SUBCOMMANDS['help'].__doc__,
+    )
+    group.add_argument(
+        '--version',
+        action='store_true',
+        help=CLI_SUBCOMMANDS['version'].__doc__,
+    )
+    group.add_argument(
+        "subcommand",
+        type=str,
+        help= "The name of the subcommand to run",
+        nargs='?',
+        choices=CLI_SUBCOMMANDS.keys(),
+        default=None,
+    )
+    parser.add_argument(
+        "subcommand_args",
+        help="Arguments for the subcommand",
+        nargs=argparse.REMAINDER,
+    )
+    command = parser.parse_args(args or ())
+
+    if command.version:
+        command.subcommand = 'version'
+    elif command.help or command.subcommand is None:
+        command.subcommand = 'help'
+
+    if command.subcommand not in ('version',):
+        from ..logging_util import log_cli_command
+
+        log_cli_command(
+            subcommand=command.subcommand,
+            subcommand_args=command.subcommand_args,
+            stdin=stdin or None,
+        )

-    sys.unraisablehook = ignore_shutdown_unraisable
    try:
-        cli(args=args, prog_name=prog_name, standalone_mode=False)
-    except click.Abort:
-        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
-        raise SystemExit(130) from None
-    except click.ClickException as err:
-        err.show()
-        raise SystemExit(err.exit_code) from None
-    except click.exceptions.Exit as err:
-        raise SystemExit(err.exit_code) from None
+        run_subcommand(
+            subcommand=command.subcommand,
+            subcommand_args=command.subcommand_args,
+            stdin=stdin or None,
+        )
    except KeyboardInterrupt:
-        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
-        raise SystemExit(130) from None
-    finally:
-        sys.unraisablehook = previous_unraisablehook
-
-
-if __name__ == "__main__":
-    main()
+        print('\n\n[red][X] Got CTRL+C. Exiting...[/red]')
@@ -1,406 +1,142 @@
 #!/usr/bin/env python3
-from __future__ import annotations

-__package__ = "archivebox.cli"
-__command__ = "archivebox add"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox add'

 import sys
-import os
-import json
-from pathlib import Path
+import argparse

-from typing import Any, TYPE_CHECKING
+from typing import List, Optional, IO

-import rich_click as click
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
+from archivebox.config.common import ARCHIVING_CONFIG

-from archivebox.config import CONSTANTS
-from archivebox.misc.util import enforce_types, docstring
-from archivebox.misc.util import parse_filesize_to_bytes
+from ..main import add
+from ..parsers import PARSERS
+from ..logging_util import SmartFormatter, accept_stdin, stderr


-if TYPE_CHECKING:
-    from django.db.models import QuerySet
-    from archivebox.crawls.models import Crawl
-    from archivebox.core.models import Snapshot
-
-
-def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
-    from archivebox.misc.jsonl import read_args_or_stdin
-    from archivebox.misc.util import validate_url
-
-    urls: list[str] = []
-    for record in read_args_or_stdin(args):
-        url = record.get("url")
-        if isinstance(url, str) and url:
-            try:
-                urls.append(validate_url(url))
-            except ValueError as err:
-                raise click.BadParameter(str(err), param_hint="URL") from err
-
-        urls_field = record.get("urls")
-        if isinstance(urls_field, str):
-            for line in urls_field.splitlines():
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    try:
-                        urls.append(validate_url(line))
-                    except ValueError as err:
-                        raise click.BadParameter(str(err), param_hint="URL") from err
-
-    return urls
-
-
-@enforce_types
-def add(
-    urls: str | list[str],
-    snapshot_ids: list[str] | None = None,
-    depth: int | str = 0,
-    max_urls: int = 0,
-    crawl_max_size: int | str = 0,
-    crawl_timeout: int = 0,
-    snapshot_max_size: int | str = 0,
-    crawl_max_concurrent_snapshots: int | None = None,
-    tag: str = "",
-    url_allowlist: str = "",
-    url_denylist: str = "",
-    parser: str = "auto",
-    plugins: str = "",
-    persona: str = "Default",
-    index_only: bool = False,
-    bg: bool = False,
-    created_by_id: int | None = None,
-    config: dict[str, Any] | None = None,
-) -> tuple[Crawl, QuerySet[Snapshot]]:
-    """Add a new URL or list of URLs to your archive.
-
-    The flow is:
-    1. Save URLs to sources file
-    2. Create Crawl with URLs and max_depth
-    3. Crawl runner creates Snapshots from Crawl URLs (depth=0)
-    4. Crawl runner runs parser extractors on root snapshots
-    5. Parser extractors output to urls.jsonl
-    6. URLs are added to Crawl.urls and child Snapshots are created
-    7. Repeat until max_depth is reached
-    """
-
-    from rich import print
-
-    depth = int(depth)
-    max_urls = int(max_urls or 0)
-    crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
-    crawl_timeout = int(crawl_timeout or 0)
-    snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    from archivebox.config.permissions import USER, HOSTNAME
-    from archivebox.config.common import get_config
-
-    config_overrides = dict(config or {})
-    runtime_config = get_config()
-    crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
-    if crawl_max_concurrent_snapshots is None:
-        crawl_max_concurrent_snapshots = runtime_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
-    crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
-
-    if depth not in (0, 1, 2, 3, 4):
-        raise ValueError("Depth must be 0-4")
-    if max_urls < 0:
-        raise ValueError("max_urls must be >= 0")
-    if crawl_max_size < 0:
-        raise ValueError("crawl_max_size must be >= 0")
-    if crawl_timeout < 0:
-        raise ValueError("crawl_timeout must be >= 0")
-    if snapshot_max_size < 0:
-        raise ValueError("snapshot_max_size must be >= 0")
-    if crawl_max_concurrent_snapshots < 1:
-        raise ValueError("crawl_max_concurrent_snapshots must be >= 1")
-
-    # import models once django is set up
-    from archivebox.crawls.models import Crawl
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.personas.models import Persona
-    from archivebox.misc.logging_util import printable_filesize
-    from archivebox.misc.system import get_dir_size
-    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-    from django.utils import timezone
-
-    created_by_id = created_by_id or get_or_create_system_user_pk()
-    started_at = timezone.now()
-
-    use_internal_input_root = isinstance(urls, str)
-    source_text = (
-        urls
-        if use_internal_input_root
-        else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 0}, separators=(",", ":")) for url in urls)
-    )
-
-    # 2. Create a new Crawl with inline URLs
-    cli_args = [*sys.argv]
-    if cli_args[0].lower().endswith("archivebox"):
-        cli_args[0] = "archivebox"
-    cmd_str = " ".join(cli_args)
-
-    timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
-
-    persona_name = (persona or "Default").strip() or "Default"
-    plugins = plugins or ""
-    persona_obj = Persona.get_or_create_named(persona_name)
-    persona_obj.ensure_dirs()
-    effective_persona_config = get_config(persona=persona_obj)
-
-    crawl_config = {
-        "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
-        **({"INDEX_ONLY": True} if index_only else {}),
-        **({"PLUGINS": plugins} if plugins else {}),
-        **(
-            {"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots}
-            if crawl_max_concurrent_snapshots_override
-            and crawl_max_concurrent_snapshots != int(effective_persona_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS)
-            else {}
-        ),
-        **({"CRAWL_MAX_URLS": max_urls} if max_urls else {}),
-        **({"CRAWL_MAX_SIZE": crawl_max_size} if crawl_max_size else {}),
-        **({"CRAWL_TIMEOUT": crawl_timeout} if crawl_timeout else {}),
-        **({"SNAPSHOT_MAX_SIZE": snapshot_max_size} if snapshot_max_size else {}),
-        **({"PARSER": parser} if parser != "auto" else {}),
-        **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
-        **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
-    }
-    # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
-    # priority for crawl-frozen keys. Runtime-derived execution keys are
-    # stripped by Crawl.save() and rederived when hooks run.
-    crawl_config.update(config_overrides)
-
-    # Crawl.urls is the user's submitted source, not a derived work queue for
-    # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
-    # callers audit or resume imports without losing RSS/Netscape/JSON metadata
-    # that is not representable as one plain URL per line.
-    crawl = Crawl.objects.create(
-        urls=source_text,
-        # Stdin/import text gets an extra hop because the synthetic
-        # archivebox://internal root lives at depth 0 and parser-discovered
-        # URLs land at depth 1; direct URL args become the depth=0 input
-        # snapshots themselves so --depth=N matches the deepest hop the user
-        # asked for.
-        max_depth=depth + 1 if use_internal_input_root else depth,
-        tags_str=tag,
-        persona_id=persona_obj.id,
-        label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
-        created_by_id=created_by_id,
-        status=Crawl.StatusChoices.QUEUED,
-        retry_at=None if index_only else timezone.now(),
-        config=crawl_config,
-    )
-
-    print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
-    first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
-    print(f"    [dim]First URL: {first_url}[/dim]")
-
-    # 3. The CrawlMachine will create Snapshots from all URLs when started
-    #    Parser extractors run on snapshots and discover more URLs
-    #    Discovered URLs become child Snapshots (depth+1)
-
-    if index_only:
-        print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
-        return crawl, crawl.snapshot_set.none()
-
-    # 5. Start the crawl runner to process the queue
-    #    The runner will:
-    #    - Process Crawl -> create Snapshots from all URLs
-    #    - Process Snapshots -> run extractors
-    #    - Parser extractors discover new URLs -> create child Snapshots
-    #    - Repeat until max_depth reached
-
-    if bg:
-        # Background mode: just queue work and return (background runner via server will pick it up).
-        print(
-            "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
-        )
-        from archivebox.services.runner import ensure_background_runner
-
-        ensure_background_runner(allow_under_pytest=True)
-    else:
-        # Foreground mode: run full crawl runner until all work is done
-        print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
-        from archivebox.machine.models import Process
-        from archivebox.core.takeover_util import command_owns_foreground_runner, current_command, standby_until_foreground_runner_needed
-        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
-
-        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
-        exit_code = 0
-        try:
-            try:
-                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
-                    while True:
-                        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
-                        exit_code = run_runner_worker(
-                            ["--crawl-id", str(crawl.id)],
-                            name=f"worker_runner_add_{os.getpid()}",
-                            interactive_interrupts=True,
-                        )
-                        crawl.refresh_from_db(fields=["status", "retry_at"])
-                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
-                            break
-                        # A shared supervisord can be stopped by another
-                        # foreground owner (for example `archivebox server`
-                        # shutting down) and stop the one-shot add runner while
-                        # this add's crawl is still runnable. Keep the add
-                        # process alive so it can re-enter the normal
-                        # foreground-runner ownership loop and finish its crawl.
-                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
-                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
-                            continue
-                        if exit_code == 0:
-                            break
-                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
-                            continue
-                        raise SystemExit(exit_code)
-            except KeyboardInterrupt:
-                exit_code = 130
-                print("\n[red][X] archivebox add interrupted.[/red]")
-                print("[yellow]Hint: resume this crawl with:[/yellow]")
-                print(f"    [green]archivebox run --crawl-id={crawl.id}[/green]")
-                raise SystemExit(exit_code)
-        finally:
-            command.mark_exited(exit_code=exit_code)
-            stop_own_supervisord_process()
-
-        # Print summary for foreground runs
-        try:
-            crawl.refresh_from_db()
-            try:
-                from django.db.models import Count, Sum
-
-                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("output_size"))
-                snapshots_count = int(totals["snapshot_count"] or 0)
-                total_bytes = int(totals["total_bytes"] or 0)
-            except Exception:
-                snapshots_count = crawl.snapshot_set.count()
-                total_bytes, _, _ = get_dir_size(crawl.output_dir)
-            total_size = printable_filesize(total_bytes)
-            total_time = timezone.now() - started_at
-            total_seconds = int(total_time.total_seconds())
-            mins, secs = divmod(total_seconds, 60)
-            hours, mins = divmod(mins, 60)
-            if hours:
-                duration_str = f"{hours}h {mins}m {secs}s"
-            elif mins:
-                duration_str = f"{mins}m {secs}s"
-            else:
-                duration_str = f"{secs}s"
-
-            # Output dir relative to DATA_DIR
-            try:
-                rel_output = Path(crawl.output_dir).relative_to(CONSTANTS.DATA_DIR)
-                rel_output_str = f"./{rel_output}"
-            except Exception:
-                rel_output_str = str(crawl.output_dir)
-
-            from archivebox.core.routes_util import build_admin_url
-
-            admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
-
-            print("\n[bold]crawl output saved to:[/bold]")
-            print(f"  {rel_output_str}")
-            print(f"  {admin_url}")
-            print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
-            print(f"[bold]total size:[/bold] {total_size}")
-            print(f"[bold]total time:[/bold] {duration_str}")
-        except Exception:
-            # Summary is best-effort; avoid failing the command if something goes wrong
-            pass
-
-    # 6. Return the list of Snapshots in this crawl
-    snapshots = crawl.snapshot_set.all()
-    return crawl, snapshots
-
-
-@click.command()
-@click.option(
-    "--depth",
-    "-d",
-    type=click.Choice([str(i) for i in range(5)]),
-    default="0",
-    help="Recursively archive linked pages up to N hops away",
-)
-@click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
-@click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
-@click.option("--crawl-timeout", type=int, default=0, help="Maximum total crawl runtime in seconds (0 = unlimited)")
-@click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)")
-@click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl")
-@click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
-@click.option("--url-allowlist", "--domain-allowlist", default="", help="Comma-separated URL/domain allowlist for this crawl")
-@click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
-@click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
-@click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
-@click.option("--extract", default="", help="Comma-separated plugins or output types to extract e.g. title,pdf,text/html,image,...")
-@click.option("--persona", default="Default", help="Authentication profile to use when archiving")
-@click.option(
-    "--only-new/--no-only-new",
-    "only_new",
-    default=None,
-    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
-    "Pass --no-only-new to force re-archive of URLs that already exist.",
-)
-@click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
-@click.option("--overwrite", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
-@click.option("--update", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
-@click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
-@click.argument("urls", nargs=-1, type=click.Path())
@docstring(add.__doc__)
-def main(**kwargs):
-    """Add a new URL or list of URLs to your archive"""
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=add.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--tag', '-t',
+        type=str,
+        default='',
+        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
+    )
+    parser.add_argument(
+        '--update', #'-u',
+        action='store_true',
+        default=not ARCHIVING_CONFIG.ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
+        help="Also retry previously skipped/failed links when adding new links",
+    )
+    parser.add_argument(
+        '--update-all', #'-n',
+        action='store_true',
+        default=False, 
+        help="Also update ALL links in index when finished adding new links",
+    )
+    parser.add_argument(
+        '--index-only', #'-o',
+        action='store_true',
+        help="Add the links to the main index without archiving them",
+    )
+    parser.add_argument(
+        'urls',
+        nargs='*',
+        type=str,
+        default=None,
+        help=(
+            'URLs or paths to archive e.g.:\n'
+            '    https://getpocket.com/users/USERNAME/feed/all\n'
+            '    https://example.com/some/rss/feed.xml\n'
+            '    https://example.com\n'
+            '    ~/Downloads/firefox_bookmarks_export.html\n'
+            '    ~/Desktop/sites_list.csv\n'
+        )
+    )
+    parser.add_argument(
+        "--depth",
+        action="store",
+        default=0,
+        choices=[0, 1],
+        type=int,
+        help="Recursively archive all linked pages up to this many hops away"
+    )
+    parser.add_argument(
+        "--overwrite",
+        default=False,
+        action="store_true",
+        help="Re-archive URLs from scratch, overwriting any existing files"
+    )
+    parser.add_argument(
+        "--init", #'-i',
+        action='store_true',
+        help="Init/upgrade the curent data directory before adding",
+    )
+    parser.add_argument(
+        "--extract",
+        type=str,
+        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
+              This does not take precedence over the configuration",
+        default=""
+    )
+    parser.add_argument(
+        "--parser",
+        type=str,
+        help="Parser used to read inputted URLs.",
+        default="auto",
+        choices=["auto", *PARSERS.keys()],
+    )
+    command = parser.parse_args(args or ())
+    urls = command.urls

-    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    stdin_urls = ''
+    if not urls:
+        stdin_urls = accept_stdin(stdin)

-    with foreground_shutdown_signals(), foreground_parent_watchdog():
-        raw_urls = kwargs.pop("urls")
-        if raw_urls:
-            urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
-        elif not sys.stdin.isatty():
-            urls = sys.stdin.read()
-        else:
-            urls = []
-        if not urls:
-            raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
-        if int(kwargs.get("max_urls") or 0) < 0:
-            raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
-        if int(kwargs.get("crawl_timeout") or 0) < 0:
-            raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout")
-        try:
-            kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
-        except ValueError as err:
-            raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
-        try:
-            kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
-        except ValueError as err:
-            raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
-        if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
-            raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
-
-        # Translate --only-new/--no-only-new into a crawl config override.
-        # add() takes config overrides as a dict; no per-flag kwargs.
-        overwrite = kwargs.pop("overwrite", False)
-        update = kwargs.pop("update", False)
-        only_new = kwargs.pop("only_new", None)
-        extract = kwargs.pop("extract", "")
-        if overwrite or update:
-            only_new = False
-        if only_new is not None:
-            kwargs["config"] = {"ONLY_NEW": bool(only_new)}
-        if extract:
-            from abx_dl.models import discover_plugins, plugins_matching_output
-
-            all_plugins = discover_plugins()
-            tokens = [token.strip() for token in extract.split(",") if token.strip()]
-            plugin_names = {name.lower(): name for name in all_plugins}
-            selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
-            selected += plugins_matching_output(all_plugins, tokens)
-            if not selected:
-                raise click.UsageError(f"No plugins found matching extract types: {extract}")
-            existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
-            kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
-
-        add(urls=urls, **kwargs)
+    if (stdin_urls and urls) or (not stdin and not urls):
+        stderr(
+            '[X] You must pass URLs/paths to add via stdin or CLI arguments.\n',
+            color='red',
+        )
+        raise SystemExit(2)
+    add(
+        urls=stdin_urls or urls,
+        depth=command.depth,
+        tag=command.tag,
+        update=command.update,
+        update_all=command.update_all,
+        index_only=command.index_only,
+        overwrite=command.overwrite,
+        init=command.init,
+        extractors=command.extract,
+        parser=command.parser,
+        out_dir=pwd or DATA_DIR,
+    )


-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
+
+
+# TODO: Implement these
+#
+# parser.add_argument(
+#     '--mirror', #'-m',
+#     action='store_true',
+#     help='Archive an entire site (finding all linked pages below it on the same domain)',
+# )
+# parser.add_argument(
+#     '--crawler', #'-r',
+#     choices=('depth_first', 'breadth_first'),
+#     help='Controls which crawler to use in order to find outlinks in a given page',
+#     default=None,
+# )
@@ -1,387 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox archiveresult <action> [args...] [--filters]
-
-Manage ArchiveResult records (plugin extraction results).
-
-Actions:
-    create  - Create ArchiveResults for Snapshots (queue extractions)
-    list    - List ArchiveResults as JSONL (with optional filters)
-    update  - Update ArchiveResults from stdin JSONL
-    delete  - Delete ArchiveResults from stdin JSONL
-
-Examples:
-    # Create ArchiveResults for snapshots (queue for extraction)
-    archivebox snapshot list --status=queued | archivebox archiveresult create
-    archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
-
-    # List with filters
-    archivebox archiveresult list --status=failed
-    archivebox archiveresult list --plugin=screenshot --status=succeeded
-
-    # Update (reset failed extractions to queued)
-    archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued
-
-    # Delete
-    archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
-
-    # Re-run failed extractions
-    archivebox archiveresult list --status=failed | archivebox run
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox archiveresult"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_util import apply_filters
-
-
-def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
-    return {
-        "type": "ArchiveResult",
-        "snapshot_id": str(snapshot_id),
-        "plugin": plugin,
-        "hook_name": hook_name,
-        "status": status,
-    }
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_archiveresults(
-    snapshot_id: str | None = None,
-    plugin: str | None = None,
-    status: str = "queued",
-) -> int:
-    """
-    Create ArchiveResult request records for Snapshots.
-
-    Reads Snapshot records from stdin and emits ArchiveResult request JSONL.
-    Pass-through: Non-Snapshot/ArchiveResult records are output unchanged.
-    If --plugin is specified, only emits requests for that plugin.
-    Otherwise, emits requests for all enabled snapshot hooks.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.config.common import get_config
-    from archivebox.plugins.hooks import discover_hooks
-    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
-    from archivebox.core.models import Snapshot
-
-    is_tty = sys.stdout.isatty()
-
-    # If snapshot_id provided directly, use that
-    if snapshot_id:
-        try:
-            snapshots = [Snapshot.objects.get(id=snapshot_id)]
-            pass_through_records = []
-        except Snapshot.DoesNotExist:
-            rprint(f"[red]Snapshot not found: {snapshot_id}[/red]", file=sys.stderr)
-            return 1
-    else:
-        # Read from stdin
-        records = list(read_stdin())
-        if not records:
-            rprint("[yellow]No Snapshot records provided via stdin[/yellow]", file=sys.stderr)
-            return 1
-
-        # Separate snapshot records from pass-through records
-        snapshot_ids = []
-        pass_through_records = []
-
-        for record in records:
-            record_type = record.get("type", "")
-
-            if record_type == TYPE_SNAPSHOT:
-                # Pass through the Snapshot record itself
-                pass_through_records.append(record)
-                if record.get("id"):
-                    snapshot_ids.append(record["id"])
-
-            elif record_type == TYPE_ARCHIVERESULT:
-                # ArchiveResult records: pass through if they have an id
-                if record.get("id"):
-                    pass_through_records.append(record)
-                # If no id, we could create it, but for now just pass through
-                else:
-                    pass_through_records.append(record)
-
-            elif record_type:
-                # Other typed records (Crawl, Tag, etc): pass through
-                pass_through_records.append(record)
-
-            elif record.get("id"):
-                # Untyped record with id - assume it's a snapshot ID
-                snapshot_ids.append(record["id"])
-
-        # Output pass-through records first
-        if not is_tty:
-            for record in pass_through_records:
-                write_record(record)
-
-        if not snapshot_ids:
-            if pass_through_records:
-                rprint(f"[dim]Passed through {len(pass_through_records)} records, no new snapshots to process[/dim]", file=sys.stderr)
-                return 0
-            rprint("[yellow]No valid Snapshot IDs in input[/yellow]", file=sys.stderr)
-            return 1
-
-        snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids))
-
-    if not snapshots:
-        rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
-        return 0 if pass_through_records else 1
-
-    created_count = 0
-    for snapshot in snapshots:
-        if plugin:
-            if not is_tty:
-                write_record(build_archiveresult_request(snapshot.id, plugin, status=status))
-            created_count += 1
-        else:
-            config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
-            hooks = discover_hooks("Snapshot", config=config)
-            for hook_path in hooks:
-                hook_name = hook_path.stem
-                plugin_name = hook_path.parent.name
-                if not is_tty:
-                    write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
-                created_count += 1
-
-    rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_archiveresults(
-    status: str | None = None,
-    plugin: str | None = None,
-    snapshot_id: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List ArchiveResults as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import ArchiveResult
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = ArchiveResult.objects.all().order_by("-start_ts")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "plugin": plugin,
-        "snapshot_id": snapshot_id,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for result in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "succeeded": "green",
-                "failed": "red",
-                "skipped": "dim",
-                "noresults": "dim",
-                "backoff": "magenta",
-            }.get(result.status, "dim")
-            rprint(
-                f"[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
-            )
-        else:
-            write_record(result.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} archive results[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_archiveresults(
-    status: str | None = None,
-) -> int:
-    """
-    Update ArchiveResults from stdin JSONL.
-
-    Reads ArchiveResult records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.core.models import ArchiveResult
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        result_id = record.get("id")
-        if not result_id:
-            continue
-
-        try:
-            result = ArchiveResult.objects.get(id=result_id)
-
-            # Apply updates from CLI flags
-            if status:
-                result.status = status
-
-            result.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(result.to_json())
-
-        except ArchiveResult.DoesNotExist:
-            rprint(f"[yellow]ArchiveResult not found: {result_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} archive results[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete ArchiveResults from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.core.models import ArchiveResult
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    result_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not result_ids:
-        rprint("[yellow]No valid archive result IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    results = ArchiveResult.objects.filter(id__in=result_ids)
-    count = results.count()
-
-    if count == 0:
-        rprint("[yellow]No matching archive results found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} archive results (dry run)[/yellow]", file=sys.stderr)
-        for result in results[:10]:
-            rprint(f"  [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}", file=sys.stderr)
-        if count > 10:
-            rprint(f"  ... and {count - 10} more", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = results.delete()
-    rprint(f"[green]Deleted {deleted_count} archive results[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage ArchiveResult records (plugin extraction results)."""
-    pass
-
-
-@main.command("create")
-@click.option("--snapshot-id", help="Snapshot ID to create results for")
-@click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
-    """Create ArchiveResults for Snapshots from stdin JSONL."""
-    sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
-
-
-@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, succeeded, failed, skipped)")
-@click.option("--plugin", "-p", help="Filter by plugin name")
-@click.option("--snapshot-id", help="Filter by snapshot ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(
-    status: str | None,
-    plugin: str | None,
-    snapshot_id: str | None,
-    limit: int | None,
-):
-    """List ArchiveResults as JSONL."""
-    sys.exit(
-        list_archiveresults(
-            status=status,
-            plugin=plugin,
-            snapshot_id=snapshot_id,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--status", "-s", help="Set status")
-def update_cmd(status: str | None):
-    """Update ArchiveResults from stdin JSONL."""
-    sys.exit(update_archiveresults(status=status))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete ArchiveResults from stdin JSONL."""
-    sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,319 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox binary <action> [args...] [--filters]
-
-Manage Binary records (detected executables like chrome, wget, etc.).
-
-Actions:
-    create  - Create/register a Binary
-    list    - List Binaries as JSONL (with optional filters)
-    update  - Update Binaries from stdin JSONL
-    delete  - Delete Binaries from stdin JSONL
-
-Examples:
-    # List all binaries
-    archivebox binary list
-
-    # List specific binary
-    archivebox binary list --name=chrome
-
-    # List binaries with specific version
-    archivebox binary list --version__icontains=120
-
-    # Delete old binary entries
-    archivebox binary list --name=chrome | archivebox binary delete --yes
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox binary"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_util import apply_filters
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_binary(
-    name: str,
-    abspath: str,
-    version: str = "",
-) -> int:
-    """
-    Create/register a Binary.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    if not name or not abspath:
-        rprint("[red]Both --name and --abspath are required[/red]", file=sys.stderr)
-        return 1
-
-    try:
-        from archivebox.machine.models import Machine
-
-        machine = Machine.current()
-        created = not Binary.objects.filter(
-            machine=machine,
-            name=name,
-            abspath=abspath,
-            version=version,
-        ).exists()
-
-        # Mirror the Binary model lifecycle used elsewhere in the system so CLI
-        # records are owned by the current machine and can be safely piped into
-        # `archivebox run` without creating invalid rows missing machine_id.
-        binary = Binary.from_json(
-            {
-                "name": name,
-                "abspath": abspath,
-                "version": version,
-                "binproviders": "env",
-                "binprovider": "env",
-            },
-        )
-        if binary is None:
-            raise ValueError("failed to create binary record")
-
-        if not is_tty:
-            write_record(binary.to_json())
-
-        if created:
-            rprint(f"[green]Created binary: {name} at {abspath}[/green]", file=sys.stderr)
-        else:
-            rprint(f"[dim]Binary already exists: {name} at {abspath}[/dim]", file=sys.stderr)
-
-        return 0
-
-    except Exception as e:
-        rprint(f"[red]Error creating binary: {e}[/red]", file=sys.stderr)
-        return 1
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_binaries(
-    name: str | None = None,
-    abspath__icontains: str | None = None,
-    version__icontains: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Binaries as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Binary.objects.all().order_by("name", "-modified_at", "-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "name": name,
-        "abspath__icontains": abspath__icontains,
-        "version__icontains": version__icontains,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for binary in queryset:
-        if is_tty:
-            rprint(f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}")
-        else:
-            write_record(binary.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} binaries[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_binaries(
-    version: str | None = None,
-    abspath: str | None = None,
-) -> int:
-    """
-    Update Binaries from stdin JSONL.
-
-    Reads Binary records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        binary_id = record.get("id")
-        if not binary_id:
-            continue
-
-        try:
-            binary = Binary.objects.get(id=binary_id)
-
-            # Apply updates from CLI flags
-            if version:
-                binary.version = version
-            if abspath:
-                binary.abspath = abspath
-
-            binary.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(binary.to_json())
-
-        except Binary.DoesNotExist:
-            rprint(f"[yellow]Binary not found: {binary_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} binaries[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_binaries(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Binaries from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.machine.models import Binary
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    binary_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not binary_ids:
-        rprint("[yellow]No valid binary IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    binaries = Binary.objects.filter(id__in=binary_ids)
-    count = binaries.count()
-
-    if count == 0:
-        rprint("[yellow]No matching binaries found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} binaries (dry run)[/yellow]", file=sys.stderr)
-        for binary in binaries:
-            rprint(f"  {binary.name} {binary.abspath}", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = binaries.delete()
-    rprint(f"[green]Deleted {deleted_count} binaries[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Binary records (detected executables)."""
-    pass
-
-
-@main.command("create")
-@click.option("--name", "-n", required=True, help="Binary name (e.g., chrome, wget)")
-@click.option("--abspath", "-p", required=True, help="Absolute path to binary")
-@click.option("--version", "-v", default="", help="Binary version")
-def create_cmd(name: str, abspath: str, version: str):
-    """Create/register a Binary."""
-    sys.exit(create_binary(name=name, abspath=abspath, version=version))
-
-
-@main.command("list")
-@click.option("--name", "-n", help="Filter by name")
-@click.option("--abspath__icontains", help="Filter by path contains")
-@click.option("--version__icontains", help="Filter by version contains")
-@click.option("--limit", type=int, help="Limit number of results")
-def list_cmd(
-    name: str | None,
-    abspath__icontains: str | None,
-    version__icontains: str | None,
-    limit: int | None,
-):
-    """List Binaries as JSONL."""
-    sys.exit(
-        list_binaries(
-            name=name,
-            abspath__icontains=abspath__icontains,
-            version__icontains=version__icontains,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--version", "-v", help="Set version")
-@click.option("--abspath", "-p", help="Set path")
-def update_cmd(version: str | None, abspath: str | None):
-    """Update Binaries from stdin JSONL."""
-    sys.exit(update_binaries(version=version, abspath=abspath))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Binaries from stdin JSONL."""
-    sys.exit(delete_binaries(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,207 +1,71 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox config'

 import sys
-import toml
-import rich_click as click
-from rich import print
+import argparse
 from pathlib import Path

-from archivebox.misc.util import docstring, enforce_types
-from archivebox.misc.toml_util import CustomTOMLEncoder
+from typing import Optional, List, IO
+
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
+from ..main import config
+from ..logging_util import SmartFormatter, accept_stdin


-def _format_toml(config: dict) -> str:
-    return toml.dumps(config, encoder=CustomTOMLEncoder()).strip().replace("\n\n", "\n")
-
-
-@enforce_types
-def config(
-    *keys,
-    get: bool = False,
-    set: bool = False,
-    search: bool = False,
-    reset: bool = False,
-    **kwargs,
-) -> None:
-    """Get and set your ArchiveBox project configuration values"""
-
-    from archivebox.misc.checks import check_data_folder
-    from archivebox.misc.logging_util import printable_config
-    from abx_plugins.plugins.base.utils import resolve_alias
-    from archivebox.config.collection import write_config_file
-    from archivebox.config import CONSTANTS_CONFIG
-    from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
-    from archivebox.plugins.discovery import discover_plugin_configs
-
-    check_data_folder()
-
-    FLAT_CONFIG = get_config().as_dict()
-    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
-    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
-    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
-    readable_config = {**writable_config, **readonly_config}
-    CONFIGS = get_all_configs()
-    plugin_schemas = {
-        plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
-    }
-    core_config_aliases = {
-        alias.upper(): field_name
-        for field_name, field in ArchiveBoxConfig.model_fields.items()
-        for alias in (field_name, str(field.alias or ""))
-        if alias
-    }
-
-    config_options: list[str] = list(kwargs.pop("key=value", []) or keys or [f"{key}={val}" for key, val in kwargs.items()])
-    no_args = not (get or set or reset or config_options)
-
-    matching_config = {}
-    if search:
-        if config_options:
-            config_options = [
-                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
-            ]
-            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
-            for config_section in CONFIGS.values():
-                aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}
-
-                for search_key in config_options:
-                    # search all aliases in the section
-                    for alias_key, key in aliases.items():
-                        if key in readable_config and search_key.lower() in alias_key.lower():
-                            matching_config[key] = dict(config_section)[key]
-
-                    # search all keys and values in the section
-                    for existing_key, value in dict(config_section).items():
-                        if existing_key in readable_config and (
-                            search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower()
-                        ):
-                            matching_config[existing_key] = value
-                    for existing_key, value in readonly_config.items():
-                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
-                            matching_config[existing_key] = value
-
-        print(printable_config(matching_config))
-        raise SystemExit(not matching_config)
-
-    elif get or no_args:
-        if config_options:
-            config_options = [
-                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
-            ]
-            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
-            failed_config = [key for key in config_options if key not in readable_config]
-            if failed_config:
-                print("\n[red][X] These options failed to get[/red]")
-                print("    {}".format("\n    ".join(config_options)))
-                raise SystemExit(1)
-        else:
-            matching_config = readable_config
-
-        # Display core config sections
-        for config_section in CONFIGS.values():
-            section_header = config_section.toml_section_header
-            if isinstance(section_header, str) and section_header:
-                print(f"[grey53]\\[{section_header}][/grey53]")
-            else:
-                print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
-
-            kv_in_section = {key: val for key, val in dict(config_section).items() if key in matching_config}
-            print(_format_toml(kv_in_section))
-            print("[grey53]################################################################[/grey53]")
-
-        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
-        if readonly_keys:
-            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
-            print(_format_toml(readonly_keys))
-            print("[grey53]################################################################[/grey53]")
-
-        plugin_keys = {}
-
-        # Collect all plugin config keys
-        for schema in plugin_schemas.values():
-            for key in schema.keys():
-                if key in matching_config and key in writable_config:
-                    plugin_keys[key] = matching_config[key]
-
-        # Display all plugin config in single [PLUGINS] section
-        if plugin_keys:
-            print("[grey53]\\[PLUGINS][/grey53]")
-            print(_format_toml(plugin_keys))
-            print("[grey53]################################################################[/grey53]")
-
-        raise SystemExit(not matching_config)
-
-    elif set:
-        new_config = {}
-        failed_options = []
-        for line in config_options:
-            if line.startswith("#") or not line.strip():
-                continue
-            if "=" not in line:
-                print("[red][X] Config KEY=VALUE must have an = sign in it[/red]")
-                print(f"    {line}")
-                raise SystemExit(2)
-
-            raw_key, val = line.split("=", 1)
-            raw_key = raw_key.upper().strip()
-            key = core_config_aliases.get(raw_key) or resolve_alias(raw_key, plugin_schemas)
-            if key != raw_key:
-                print(
-                    f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
-                )
-
-            if key in writable_config:
-                new_config[key] = val.strip()
-            else:
-                failed_options.append(line)
-
-        if new_config:
-            before = FLAT_CONFIG
-            matching_config = write_config_file(new_config)
-            after = get_config().as_dict()
-            print(printable_config(matching_config))
-
-            side_effect_changes = {}
-            for key, val in after.items():
-                if key in FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
-                    side_effect_changes[key] = after[key]
-
-            if side_effect_changes:
-                print(file=sys.stderr)
-                print("[yellow][i] Note: This change also affected these other options that depended on it:[/yellow]", file=sys.stderr)
-                print("    {}".format(printable_config(side_effect_changes, prefix="    ")), file=sys.stderr)
-
-        if failed_options:
-            print()
-            print("[red][X] These options failed to set (check for typos):[/red]")
-            print("    {}".format("\n    ".join(failed_options)))
-            raise SystemExit(1)
-
-    elif reset:
-        print("[red][X] This command is not implemented yet.[/red]")
-        print("    Please manually remove the relevant lines from your config file:")
-        raise SystemExit(2)
-
-    else:
-        print("[red][X] You must pass either --get or --set, or no arguments to get the whole config.[/red]")
-        print("    archivebox config")
-        print("    archivebox config --get SOME_KEY")
-        print("    archivebox config --set SOME_KEY=SOME_VALUE")
-        raise SystemExit(2)
-
-
-@click.command()
-@click.option("--search", is_flag=True, help="Search config KEYs, VALUEs, and ALIASES for the given term")
-@click.option("--get", is_flag=True, help="Get the value for the given config KEYs")
-@click.option("--set", is_flag=True, help="Set the given KEY=VALUE config values")
-@click.option("--reset", is_flag=True, help="Reset the given KEY config values to their defaults")
-@click.argument("KEY=VALUE", nargs=-1, type=str)
@docstring(config.__doc__)
-def main(**kwargs) -> None:
-    config(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=config.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    group = parser.add_mutually_exclusive_group()
+    parser.add_argument(
+        '--search',
+        action='store_true',
+        help="Search config KEYs, VALUEs, and ALIASES for the given term",
+    )
+    group.add_argument(
+        '--get', #'-g',
+        action='store_true',
+        help="Get the value for the given config KEYs",
+    )
+    group.add_argument(
+        '--set', #'-s',
+        action='store_true',
+        help="Set the given KEY=VALUE config values",
+    )
+    group.add_argument(
+        '--reset', #'-s',
+        action='store_true',
+        help="Reset the given KEY config values to their defaults",
+    )
+    parser.add_argument(
+        'config_options',
+        nargs='*',
+        type=str,
+        help='KEY or KEY=VALUE formatted config values to get or set',
+    )
+    command = parser.parse_args(args or ())
+
+    config_options_str = ''
+    if not command.config_options:
+        config_options_str = accept_stdin(stdin)
+
+    config(
+        config_options_str=config_options_str,
+        config_options=command.config_options,
+        search=command.search,
+        get=command.get,
+        set=command.set,
+        reset=command.reset,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
+    )


-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,406 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox crawl <action> [args...] [--filters]
-
-Manage Crawl records.
-
-Actions:
-    create  - Create Crawl jobs from URLs
-    list    - List Crawls as JSONL (with optional filters)
-    update  - Update Crawls from stdin JSONL
-    delete  - Delete Crawls from stdin JSONL
-
-Examples:
-    # Create
-    archivebox crawl create https://example.com https://foo.com --depth=1
-    archivebox crawl create --tag=news https://example.com
-
-    # List with filters
-    archivebox crawl list --status=queued
-    archivebox crawl list --urls__icontains=example.com
-
-    # Update
-    archivebox crawl list --status=started | archivebox crawl update --status=queued
-
-    # Delete
-    archivebox crawl list --urls__icontains=spam.com | archivebox crawl delete --yes
-
-    # Full pipeline
-    archivebox crawl create https://example.com | archivebox snapshot create | archivebox run
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-from collections.abc import Iterable
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_util import apply_filters
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_crawl(
-    urls: Iterable[str],
-    depth: int = 0,
-    tag: str = "",
-    status: str = "queued",
-    created_by_id: int | None = None,
-) -> int:
-    """
-    Create a Crawl job from URLs.
-
-    Takes URLs as args or stdin, creates one Crawl with all URLs, outputs JSONL.
-    Pass-through: Records that are not URLs are output unchanged (for piping).
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
-    from archivebox.misc.util import validate_url
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.crawls.models import Crawl
-
-    created_by_id = created_by_id or get_or_create_system_user_pk()
-    is_tty = sys.stdout.isatty()
-
-    # Collect all input records
-    records = list(read_args_or_stdin(urls))
-
-    if not records:
-        rprint("[yellow]No URLs provided. Pass URLs as arguments or via stdin.[/yellow]", file=sys.stderr)
-        return 1
-
-    # Separate pass-through records from URL records
-    url_list = []
-    pass_through_records = []
-
-    for record in records:
-        record_type = record.get("type", "")
-
-        # Pass-through: output records that aren't URL/Crawl types
-        if record_type and record_type != TYPE_CRAWL and not record.get("url") and not record.get("urls"):
-            pass_through_records.append(record)
-            continue
-
-        # Handle existing Crawl records (just pass through with id)
-        if record_type == TYPE_CRAWL and record.get("id"):
-            pass_through_records.append(record)
-            continue
-
-        # Collect URLs
-        url = record.get("url")
-        if url:
-            try:
-                url_list.append(validate_url(str(url)))
-            except ValueError as err:
-                rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
-                return 1
-
-        # Handle 'urls' field (newline-separated)
-        urls_field = record.get("urls")
-        if urls_field:
-            for line in urls_field.split("\n"):
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    try:
-                        url_list.append(validate_url(line))
-                    except ValueError as err:
-                        rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
-                        return 1
-
-    # Output pass-through records first
-    if not is_tty:
-        for record in pass_through_records:
-            write_record(record)
-
-    if not url_list:
-        if pass_through_records:
-            # If we had pass-through records but no URLs, that's OK
-            rprint(f"[dim]Passed through {len(pass_through_records)} records, no new URLs[/dim]", file=sys.stderr)
-            return 0
-        rprint("[red]No valid URLs found[/red]", file=sys.stderr)
-        return 1
-
-    try:
-        # Build crawl record with all URLs as newline-separated string
-        crawl_record = {
-            "urls": "\n".join(url_list),
-            "max_depth": depth,
-            "tags_str": tag,
-            "status": status,
-            "label": "",
-        }
-
-        crawl = Crawl.from_json(crawl_record, overrides={"created_by_id": created_by_id})
-        if not crawl:
-            rprint("[red]Failed to create crawl[/red]", file=sys.stderr)
-            return 1
-
-        # Output JSONL record (only when piped)
-        if not is_tty:
-            write_record(crawl.to_json())
-
-        rprint(f"[green]Created crawl with {len(url_list)} URLs[/green]", file=sys.stderr)
-
-        # If TTY, show human-readable output
-        if is_tty:
-            rprint(f"  [dim]{crawl.id}[/dim]", file=sys.stderr)
-            for url in url_list[:5]:  # Show first 5 URLs
-                rprint(f"    {url[:70]}", file=sys.stderr)
-            if len(url_list) > 5:
-                rprint(f"    ... and {len(url_list) - 5} more", file=sys.stderr)
-
-        return 0
-
-    except Exception as e:
-        rprint(f"[red]Error creating crawl: {e}[/red]", file=sys.stderr)
-        return 1
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_crawls(
-    status: str | None = None,
-    urls__icontains: str | None = None,
-    max_depth: int | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Crawls as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.crawls.models import Crawl
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Crawl.objects.all().order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "urls__icontains": urls__icontains,
-        "max_depth": max_depth,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for crawl in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(crawl.status, "dim")
-            url_preview = crawl.urls[:50].replace("\n", " ")
-            rprint(f"[{status_color}]{crawl.status:8}[/{status_color}] [dim]{crawl.id}[/dim] {url_preview}...")
-        else:
-            write_record(crawl.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} crawls[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_crawls(
-    status: str | None = None,
-    max_depth: int | None = None,
-) -> int:
-    """
-    Update Crawls from stdin JSONL.
-
-    Reads Crawl records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from django.utils import timezone
-
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.crawls.models import Crawl
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        crawl_id = record.get("id")
-        if not crawl_id:
-            continue
-
-        try:
-            crawl = Crawl.objects.get(id=crawl_id)
-
-            if status:
-                if status not in Crawl.StatusChoices.values:
-                    rprint(f"[red]Invalid crawl status: {status}[/red]", file=sys.stderr)
-                    continue
-                if status == Crawl.StatusChoices.SEALED:
-                    crawl.cancel()
-                elif status == Crawl.StatusChoices.PAUSED:
-                    crawl.pause()
-                elif status == Crawl.StatusChoices.QUEUED:
-                    if crawl.status == Crawl.StatusChoices.PAUSED:
-                        crawl.resume()
-                    else:
-                        crawl.update_and_requeue(status=Crawl.StatusChoices.QUEUED, retry_at=timezone.now())
-                elif status == Crawl.StatusChoices.STARTED:
-                    crawl.update_and_requeue(status=Crawl.StatusChoices.STARTED, retry_at=timezone.now())
-            if max_depth is not None:
-                crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
-                crawl.max_depth = max_depth
-            elif not status:
-                crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
-            updated_count += 1
-
-            if not is_tty:
-                crawl.refresh_from_db()
-                write_record(crawl.to_json())
-
-        except Crawl.DoesNotExist:
-            rprint(f"[yellow]Crawl not found: {crawl_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} crawls[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_crawls(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Crawls from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.crawls.models import Crawl
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    crawl_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not crawl_ids:
-        rprint("[yellow]No valid crawl IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    crawls = Crawl.objects.filter(id__in=crawl_ids)
-    count = crawls.count()
-
-    if count == 0:
-        rprint("[yellow]No matching crawls found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} crawls (dry run)[/yellow]", file=sys.stderr)
-        for crawl in crawls:
-            url_preview = crawl.urls[:50].replace("\n", " ")
-            rprint(f"  [dim]{crawl.id}[/dim] {url_preview}...", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = crawls.delete()
-    rprint(f"[green]Deleted {deleted_count} crawls[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Crawl records."""
-    pass
-
-
-@main.command("create")
-@click.argument("urls", nargs=-1)
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(urls: tuple, depth: int, tag: str, status: str):
-    """Create a Crawl job from URLs or stdin."""
-    sys.exit(create_crawl(urls, depth=depth, tag=tag, status=status))
-
-
-@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--urls__icontains", help="Filter by URLs contains")
-@click.option("--max-depth", type=int, help="Filter by max depth")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(
-    status: str | None,
-    urls__icontains: str | None,
-    max_depth: int | None,
-    limit: int | None,
-):
-    """List Crawls as JSONL."""
-    sys.exit(
-        list_crawls(
-            status=status,
-            urls__icontains=urls__icontains,
-            max_depth=max_depth,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--status", "-s", help="Set status")
-@click.option("--max-depth", type=int, help="Set max depth")
-def update_cmd(status: str | None, max_depth: int | None):
-    """Update Crawls from stdin JSONL."""
-    sys.exit(update_crawls(status=status, max_depth=max_depth))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Crawls from stdin JSONL."""
-    sys.exit(delete_crawls(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,465 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox extract [snapshot_ids...] [--plugins=NAMES]
-
-Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL.
-
-Input formats:
-    - Snapshot UUIDs (one per line)
-    - JSONL: {"type": "Snapshot", "id": "...", "url": "..."}
-    - JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."}
-
-Output (JSONL):
-    {"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."}
-
-Examples:
-    # Extract specific snapshot
-    archivebox extract 01234567-89ab-cdef-0123-456789abcdef
-
-    # Pipe from snapshot command
-    archivebox snapshot https://example.com | archivebox extract
-
-    # Run specific plugins only
-    archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef
-
-    # Chain commands
-    archivebox crawl https://example.com | archivebox snapshot | archivebox extract
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox extract"
-
-import sys
-from collections import defaultdict
-from itertools import product
-
-import rich_click as click
-
-
-def process_archiveresult_by_id(archiveresult_id: str) -> int:
-    """
-    Re-run extraction for a single ArchiveResult by ID.
-
-    ArchiveResults are projected status rows, not queued work items. Re-running
-    a single result means resetting that row and queueing its parent snapshot
-    through the shared crawl runner with the corresponding plugin selected.
-    """
-    from rich import print as rprint
-    from archivebox.core.models import ArchiveResult
-    from archivebox.api.v1_core import _uuid_ref_query
-    from archivebox.services.runner import run_crawl
-
-    try:
-        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
-    except ArchiveResult.DoesNotExist:
-        rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
-        return 1
-
-    rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
-
-    try:
-        archiveresult.reset_for_retry()
-        snapshot = archiveresult.snapshot
-        snapshot.queue_for_extraction()
-        crawl = snapshot.crawl
-        if not crawl.claim_processing_lock(lock_seconds=10):
-            rprint(
-                f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]",
-                file=sys.stderr,
-            )
-            return 1
-
-        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
-        archiveresult.refresh_from_db()
-
-        if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
-            print(f"[green]Extraction succeeded: {archiveresult.output_str}[/green]")
-            return 0
-        elif archiveresult.status == ArchiveResult.StatusChoices.NORESULTS:
-            print(f"[dim]Extraction completed with no results: {archiveresult.output_str}[/dim]")
-            return 0
-        elif archiveresult.status == ArchiveResult.StatusChoices.FAILED:
-            print(f"[red]Extraction failed: {archiveresult.output_str}[/red]", file=sys.stderr)
-            return 1
-        else:
-            # Still in progress or backoff - not a failure
-            print(f"[yellow]Extraction status: {archiveresult.status}[/yellow]")
-            return 0
-
-    except Exception as e:
-        print(f"[red]Extraction error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-        return 1
-
-
-def run_plugins(
-    args: tuple,
-    records: list[dict] | None = None,
-    plugins: str = "",
-    wait: bool = True,
-    emit_results: bool = True,
-    show_progress: bool = True,
-    preserve_queued: bool = False,
-) -> int:
-    """
-    Run plugins on Snapshots from input.
-
-    Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from rich import print as rprint
-    from django.utils import timezone
-
-    from archivebox.misc.jsonl import (
-        read_args_or_stdin,
-        write_record,
-        TYPE_SNAPSHOT,
-        TYPE_ARCHIVERESULT,
-    )
-    from archivebox.core.models import Snapshot
-    from archivebox.core.models import ArchiveResult
-    from archivebox.services.runner import run_crawl
-    from abx_dl.models import discover_plugins
-
-    is_tty = sys.stdout.isatty()
-
-    # Parse comma-separated plugins list once (reused in creation and filtering)
-    plugins_list = [p.strip() for p in plugins.split(",") if p.strip()] if plugins else []
-
-    # Parse stdin/args exactly once per CLI invocation.
-    # `main()` may already have consumed stdin to distinguish Snapshot input from
-    # ArchiveResult IDs; if so, it must pass the parsed records through here
-    # instead of asking this helper to reread an already-drained pipe.
-    if records is None:
-        records = list(read_args_or_stdin(args))
-
-    if not records:
-        rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
-        return 1
-
-    # Gather snapshot IDs and optional plugin constraints to process
-    snapshot_ids = set()
-    requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set)
-    for record in records:
-        record_type = record.get("type")
-
-        if record_type == TYPE_SNAPSHOT:
-            snapshot_id = record.get("id")
-            if snapshot_id:
-                snapshot_ids.add(str(snapshot_id))
-            elif record.get("url"):
-                # Look up by URL (get most recent if multiple exist)
-                snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
-                if snap:
-                    snapshot_ids.add(str(snap.id))
-                else:
-                    rprint(f"[yellow]Snapshot not found for URL: {record['url']}[/yellow]", file=sys.stderr)
-
-        elif record_type == TYPE_ARCHIVERESULT:
-            snapshot_id = record.get("snapshot_id")
-            if snapshot_id:
-                snapshot_ids.add(str(snapshot_id))
-                plugin_name = record.get("plugin")
-                if plugin_name and not plugins_list:
-                    requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
-
-        elif "id" in record:
-            # Assume it's a snapshot ID
-            snapshot_ids.add(str(record["id"]))
-
-    if not snapshot_ids:
-        rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
-        return 1
-
-    existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id"))
-    existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots}
-    existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots}
-    missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids)
-    for snapshot_id in missing_snapshot_ids:
-        rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
-
-    # Queue only the target plugin rows. Bulk updates keep large reindex runs
-    # from doing one SELECT+UPDATE per snapshot/plugin before hooks even start.
-    requested_pairs: set[tuple[str, str]] = set()
-    if plugins_list:
-        requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list))
-    else:
-        requested_pairs.update(
-            (snapshot_id, plugin_name)
-            for snapshot_id, plugin_names in requested_plugins_by_snapshot.items()
-            if snapshot_id in existing_snapshot_ids
-            for plugin_name in plugin_names
-        )
-    plugins_by_name = discover_plugins(runtime="archivebox")
-    requested_rows: set[tuple[str, str, str]] = set()
-    for snapshot_id, plugin_name in requested_pairs:
-        plugin = plugins_by_name.get(plugin_name)
-        hooks = plugin.filter_hooks("Snapshot") if plugin is not None else []
-        if hooks:
-            requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks)
-        else:
-            requested_rows.add((snapshot_id, plugin_name, ""))
-
-    queued_rows: set[tuple[str, str, str]] = set()
-    if preserve_queued and requested_rows:
-        queued_rows = {
-            (str(snapshot_id), plugin_name, hook_name)
-            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
-                snapshot_id__in=existing_snapshot_ids,
-                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
-                status=ArchiveResult.StatusChoices.QUEUED,
-            ).values_list("snapshot_id", "plugin", "hook_name")
-        }
-    rows_to_queue = requested_rows - queued_rows
-
-    reset_fields = {
-        "status": ArchiveResult.StatusChoices.QUEUED,
-        "output_str": "",
-        "output_json": None,
-        "output_files": {},
-        "output_size": 0,
-        "output_mimetypes": "",
-        "start_ts": None,
-        "end_ts": None,
-        "modified_at": timezone.now(),
-    }
-    if rows_to_queue and plugins_list:
-        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
-        for snapshot_id, plugin_name, hook_name in rows_to_queue:
-            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
-        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
-                **reset_fields,
-            )
-    elif rows_to_queue and requested_plugins_by_snapshot:
-        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
-        for snapshot_id, plugin_name, hook_name in rows_to_queue:
-            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
-        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
-                **reset_fields,
-            )
-    existing_rows = (
-        {
-            (str(snapshot_id), plugin_name, hook_name)
-            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
-                snapshot_id__in=existing_snapshot_ids,
-                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
-            ).values_list("snapshot_id", "plugin", "hook_name")
-        }
-        if rows_to_queue
-        else set()
-    )
-    missing_rows = rows_to_queue - existing_rows
-    if missing_rows:
-        ArchiveResult.objects.bulk_create(
-            [
-                ArchiveResult(
-                    snapshot_id=snapshot_id,
-                    plugin=plugin_name,
-                    hook_name=hook_name,
-                    status=ArchiveResult.StatusChoices.QUEUED,
-                )
-                for snapshot_id, plugin_name, hook_name in sorted(missing_rows)
-            ],
-            batch_size=500,
-        )
-
-    processed_count = len(existing_snapshot_ids)
-    queue_at = timezone.now()
-    if existing_snapshot_ids:
-        if requested_rows:
-            # Search indexing on a sealed Snapshot is the only targeted hook
-            # allowed to bypass the normal lifecycle. Every other requested
-            # plugin requeues its Snapshot through the unified state machine.
-            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
-            if preserve_queued and queued_rows:
-                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
-                affected_snapshot_ids.update(
-                    str(snapshot_id)
-                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
-                    .filter(retry_at__gt=queue_at)
-                    .values_list("id", flat=True)
-                )
-                affected_snapshot_ids.update(
-                    str(snapshot_id)
-                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
-                        "id",
-                        flat=True,
-                    )
-                )
-            requested_plugins_by_id: dict[str, set[str]] = defaultdict(set)
-            for snapshot_id, plugin_name, _hook_name in requested_rows:
-                requested_plugins_by_id[snapshot_id].add(plugin_name)
-            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
-                # Guard the read-time status so we never bump retry_at on a
-                # row that's been re-queued / started by a concurrent runner.
-                plugin_names = requested_plugins_by_id.get(str(snapshot.id), set())
-                sealed_search_backfill = (
-                    snapshot.status == Snapshot.StatusChoices.SEALED
-                    and plugin_names
-                    and all(plugin_name.startswith("search_backend_") for plugin_name in plugin_names)
-                )
-                if sealed_search_backfill:
-                    snapshot.safe_update(
-                        {"retry_at": queue_at, "modified_at": queue_at},
-                        refresh=False,
-                        extra_filter={"status": snapshot.status},
-                    )
-                else:
-                    snapshot.update_and_requeue(
-                        status=Snapshot.StatusChoices.QUEUED,
-                        retry_at=queue_at,
-                        current_step=0,
-                    )
-        else:
-            # No plugin rows were requested, so this is a full snapshot retry.
-            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):
-                snapshot.safe_update(
-                    {
-                        "status": Snapshot.StatusChoices.QUEUED,
-                        "retry_at": queue_at,
-                        "current_step": 0,
-                        "modified_at": queue_at,
-                    },
-                    refresh=False,
-                    extra_filter={"status": snapshot.status},
-                )
-    if existing_crawl_ids and not requested_rows:
-        from archivebox.crawls.models import Crawl
-
-        for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"):
-            update_fields = {
-                "retry_at": queue_at,
-                "modified_at": queue_at,
-            }
-            if crawl.status != Crawl.StatusChoices.STARTED:
-                update_fields["status"] = Crawl.StatusChoices.QUEUED
-            crawl.safe_update(
-                update_fields,
-                refresh=False,
-                extra_filter={"status": crawl.status},
-            )
-
-    if processed_count == 0:
-        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
-        return 1
-
-    if show_progress:
-        rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
-
-    # Run orchestrator if --wait (default)
-    if wait:
-        if show_progress:
-            rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
-        snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, crawl_id in existing_snapshots:
-            snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id))
-
-        for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
-            from archivebox.crawls.models import Crawl
-
-            crawl = Crawl.objects.get(id=crawl_id)
-            if not crawl.claim_processing_lock(lock_seconds=10):
-                rprint(
-                    f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]",
-                    file=sys.stderr,
-                )
-                return 1
-            selected_plugins = (
-                plugins_list
-                or sorted(
-                    {plugin for snapshot_id in crawl_snapshot_ids for plugin in requested_plugins_by_snapshot.get(str(snapshot_id), set())},
-                )
-                or None
-            )
-            run_crawl(
-                crawl_id,
-                snapshot_ids=sorted(crawl_snapshot_ids),
-                selected_plugins=selected_plugins,
-                show_progress=show_progress,
-            )
-
-    if not emit_results:
-        return 0
-
-    # Output results as JSONL (when piped) or human-readable (when TTY)
-    for snapshot_id in snapshot_ids:
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-            results = snapshot.archiveresult_set.all()
-            if plugins_list:
-                results = results.filter(plugin__in=plugins_list)
-
-            for result in results:
-                if is_tty:
-                    status_color = {
-                        "succeeded": "green",
-                        "failed": "red",
-                        "skipped": "yellow",
-                    }.get(result.status, "dim")
-                    rprint(
-                        f"  [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ''}",
-                        file=sys.stderr,
-                    )
-                else:
-                    write_record(result.to_json())
-        except Snapshot.DoesNotExist:
-            continue
-
-    return 0
-
-
-def is_archiveresult_id(value: str) -> bool:
-    """Check if value resolves to an ArchiveResult ID."""
-    from archivebox.core.models import ArchiveResult
-    from archivebox.api.v1_core import _uuid_ref_query
-
-    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()
-
-
-@click.command()
-@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run (e.g., screenshot,singlefile)")
-@click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
-@click.argument("args", nargs=-1)
-def main(plugins: str, wait: bool, args: tuple):
-    """Run plugins on Snapshots, or process existing ArchiveResults by ID"""
-    from archivebox.misc.jsonl import read_args_or_stdin
-
-    # Read all input
-    records = list(read_args_or_stdin(args))
-
-    if not records:
-        from rich import print as rprint
-
-        rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
-        sys.exit(1)
-
-    # Check if input looks like existing ArchiveResult IDs to process
-    all_are_archiveresult_ids = all(is_archiveresult_id(r.get("id") or r.get("url", "")) for r in records)
-
-    if all_are_archiveresult_ids:
-        # Process existing ArchiveResults by ID
-        from rich import print as rprint
-
-        exit_code = 0
-        for record in records:
-            archiveresult_id = record.get("id") or record.get("url")
-            if not isinstance(archiveresult_id, str):
-                rprint(f"[red]Invalid ArchiveResult input: {record}[/red]", file=sys.stderr)
-                exit_code = 1
-                continue
-            result = process_archiveresult_by_id(archiveresult_id)
-            if result != 0:
-                exit_code = result
-        sys.exit(exit_code)
-    else:
-        # Default behavior: run plugins on Snapshots from input
-        sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,154 +1,32 @@
 #!/usr/bin/env python3
-__package__ = "archivebox.cli"
-__command__ = "archivebox help"

-import ast
-import importlib.util
-import os
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox help'
+
+import sys
+import argparse
 from pathlib import Path
+from typing import Optional, List, IO

-import click
-from rich import print
-from rich.panel import Panel
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
+from ..main import help
+from ..logging_util import SmartFormatter, reject_stdin


-def _command_doc(cmd: str, import_path: str) -> str:
-    def first_doc_line(docstring: str | None) -> str:
-        lines = (docstring or "").splitlines()
-        return lines[0] if lines else ""
-
-    modname, _ = import_path.rsplit(".", 1)
-    spec = importlib.util.find_spec(modname)
-    if spec is None or spec.origin is None:
-        return ""
-
-    try:
-        tree = ast.parse(Path(spec.origin).read_text())
-    except OSError:
-        return ""
-
-    for name in (cmd, "main"):
-        for node in tree.body:
-            if isinstance(node, ast.FunctionDef) and node.name == name:
-                return first_doc_line(ast.get_docstring(node))
-
-    return first_doc_line(ast.get_docstring(tree))
-
-
-def help() -> None:
-    """Print the ArchiveBox help message and usage"""
-
-    from archivebox.cli import ArchiveBoxGroup
-    from archivebox.config import CONSTANTS
-    from archivebox.config.common import get_config
-    from archivebox.config.permissions import IN_DOCKER
-    from archivebox.misc.logging_util import log_cli_command
-
-    log_cli_command("help", [], None, ".")
-
-    COMMANDS_HELP_TEXT = (
-        "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.meta_commands.items())
-        + "\n\n    "
-        + "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.setup_commands.items())
-        + "\n\n    "
-        + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.archive_commands.items()
-        )
+@docstring(help.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=help.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
    )
-
-    DOCKER_USAGE = (
-        """
-[dodger_blue3]Docker Usage:[/dodger_blue3]
-    [grey53]# using Docker Compose:[/grey53]
-    [blue]docker compose run[/blue] [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-
-    [grey53]# using Docker:[/grey53]
-    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 8000:8000[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-"""
-        if IN_DOCKER
-        else ""
-    )
-    DOCKER_DOCS = (
-        "\n    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker[/link]"
-        if IN_DOCKER
-        else ""
-    )
-    DOCKER_OUTSIDE_HINT = "\n    [grey53]# outside of Docker:[/grey53]" if IN_DOCKER else ""
-    DOCKER_CMD_PREFIX = "[blue]docker ... [/blue]" if IN_DOCKER else ""
-
-    print(f"""{DOCKER_USAGE}
-[deep_sky_blue4]Usage:[/deep_sky_blue4]{DOCKER_OUTSIDE_HINT}
-    [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-
-[deep_sky_blue4]Commands:[/deep_sky_blue4]
-    {COMMANDS_HELP_TEXT}
-
-[deep_sky_blue4]Documentation:[/deep_sky_blue4]
-    [link=https://github.com/ArchiveBox/ArchiveBox/wiki]https://github.com/ArchiveBox/ArchiveBox/wiki[/link]{DOCKER_DOCS}
-    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage[/link]
-    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
-""")
-
-    get_config()
-    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
-        pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
-        EXAMPLE_USAGE = f"""
-[light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]
-
-[violet]Hint:[/violet] [i]Common maintenance tasks:[/i]
-    [dark_green]archivebox[/dark_green] [green]init[/green]      [grey53]# make sure database is up-to-date (safe to run multiple times)[/grey53]
-    [dark_green]archivebox[/dark_green] [green]install[/green]   [grey53]# make sure plugins are up-to-date (wget, chrome, singlefile, etc.)[/grey53]
-    [dark_green]archivebox[/dark_green] [green]status[/green]    [grey53]# get a health checkup report on your collection[/grey53]
-    [dark_green]archivebox[/dark_green] [green]update[/green]    [grey53]# retry any previously failed or interrupted archiving tasks[/grey53]
-
-[violet]Hint:[/violet] [i]More example usage:[/i]
-    [dark_green]archivebox[/dark_green] [green]add[/green] --depth=1 "https://example.com/some/page"
-    [dark_green]archivebox[/dark_green] [green]list[/green] --sort=timestamp --csv=timestamp,downloaded_at,url,title
-    [dark_green]archivebox[/dark_green] [green]schedule[/green] --every=day --depth=1 "https://example.com/some/feed.rss"
-    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:8000[/blue]                [grey53]# Start the Web UI / API server[/grey53]
-"""
-        print(
-            Panel(
-                EXAMPLE_USAGE,
-                expand=False,
-                border_style="grey53",
-                title="[green3]:white_check_mark: A collection [light_slate_blue]DATA DIR[/light_slate_blue] is currently active[/green3]",
-                subtitle="Commands run inside this dir will only apply to this collection.",
-            ),
-        )
-    else:
-        DATA_SETUP_HELP = "\n"
-        if IN_DOCKER:
-            DATA_SETUP_HELP += "[violet]Hint:[/violet] When using Docker, you need to mount a volume to use as your data dir:\n"
-            DATA_SETUP_HELP += "    docker run [violet]-v /some/path/data:/data[/violet] archivebox/archivebox ...\n\n"
-        DATA_SETUP_HELP += "To load an [dark_blue]existing[/dark_blue] collection:\n"
-        DATA_SETUP_HELP += "    1. [green]cd[/green] ~/archivebox/data     [grey53]# go into existing [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
-        DATA_SETUP_HELP += f"    2. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# migrate to latest version (safe to run multiple times)[/grey53]\n"
-        DATA_SETUP_HELP += f"    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-update all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
-        DATA_SETUP_HELP += f"    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ...get help with next steps... [/grey53]\n\n"
-        DATA_SETUP_HELP += "To start a [sea_green1]new[/sea_green1] collection:\n"
-        DATA_SETUP_HELP += "    1. [green]mkdir[/green] ~/archivebox/data  [grey53]# create a new, empty [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
-        DATA_SETUP_HELP += "    2. [green]cd[/green] ~/archivebox/data     [grey53]# cd into the new directory[/grey53]\n"
-        DATA_SETUP_HELP += f"    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# initialize ArchiveBox in the new data dir[/grey53]\n"
-        DATA_SETUP_HELP += f"    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-install all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
-        DATA_SETUP_HELP += f"    5. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ... get help with next steps... [/grey53]\n"
-        print(
-            Panel(
-                DATA_SETUP_HELP,
-                expand=False,
-                border_style="grey53",
-                title="[red]:cross_mark: No collection is currently active[/red]",
-                subtitle="All archivebox [green]commands[/green] should be run from inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
-            ),
-        )
+    parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)
+    
+    help(out_dir=Path(pwd) if pwd else DATA_DIR)


-@click.command()
-@click.option("--help", "-h", is_flag=True, help="Show help")
-def main(**kwargs):
-    """Print the ArchiveBox help message and usage"""
-    return help()
-
-
-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,196 +1,57 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox init'

-import os
 import sys
-from pathlib import Path
+import argparse

-from rich import print
-import rich_click as click
+from typing import Optional, List, IO

-from archivebox.misc.util import docstring, enforce_types
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
+from ..logging_util import SmartFormatter, reject_stdin
+from ..main import init


-def _display_data_path(path: Path, data_dir: Path) -> str:
-    path = Path(path).resolve()
-    data_dir = Path(data_dir).resolve()
-    try:
-        return f"./{path.relative_to(data_dir)}"
-    except ValueError:
-        return str(path)
-
-
-@enforce_types
-def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
-    """Initialize a new ArchiveBox collection in the current directory"""
-
-    from archivebox.config import CONSTANTS, VERSION
-    from archivebox.config.common import get_config
-    from archivebox.config.collection import write_config_file
-    from archivebox.misc.db import apply_migrations
-    from archivebox.misc.checks import check_migrations
-
-    config = get_config()
-
-    # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
-    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
-    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
-
-    is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
-    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
-    if is_empty and not existing_index:
-        print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
-        print("[green]----------------------------------------------------------------------[/green]")
-    elif existing_index:
-        # TODO: properly detect and print the existing version in current index as well
-        print(f"[green][*] Verifying and updating existing ArchiveBox collection to v{VERSION}...[/green]")
-        print("[green]----------------------------------------------------------------------[/green]")
-    else:
-        if force:
-            print("[red][!] This folder appears to already have files in it, but no index.sqlite3 is present.[/red]")
-            print("[red]    Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).[/red]")
-        else:
-            print(
-                "[red][X] This folder appears to already have files in it, but no index.sqlite3 present.[/red]\n\n"
-                "    You must run init in a completely empty directory, or an existing data folder.\n\n"
-                "    [violet]Hint:[/violet] To import an existing data folder make sure to cd into the folder first, \n"
-                "    then run and run 'archivebox init' to pick up where you left off.\n\n"
-                "    (Always make sure your data folder is backed up first before updating ArchiveBox)",
-            )
-            raise SystemExit(2)
-
-    if existing_index:
-        print("\n[green][*] Verifying archive folder structure...[/green]")
-    else:
-        print("\n[green][+] Building archive folder structure...[/green]")
-
-    archive_path = _display_data_path(CONSTANTS.ARCHIVE_DIR, CONSTANTS.DATA_DIR)
-    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, CONSTANTS.DATA_DIR)
-    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, CONSTANTS.DATA_DIR)
-    print(f"    + {archive_path}, {sources_path}, {logs_path}...")
-    Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
-    CONSTANTS.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
-    CONSTANTS.USERS_DIR.mkdir(parents=True, exist_ok=True)
-    Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
-    for path in (Path(CONSTANTS.SOURCES_DIR), CONSTANTS.ARCHIVE_DIR, CONSTANTS.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
-        path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)
-
-    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, CONSTANTS.DATA_DIR)}...")
-
-    # create the .archivebox_id file with a unique ID for this collection
-    from archivebox.config.paths import _get_collection_id
-
-    _get_collection_id(CONSTANTS.DATA_DIR, force_create=True)
-
-    # create the ArchiveBox.conf file
-    write_config_file({"SECRET_KEY": config.SECRET_KEY})
-
-    if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
-        print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]")
-    else:
-        print("\n[green][+] Building main SQL index and running initial migrations...[/green]")
-
-    from archivebox.config.django import setup_django
-
-    setup_django()
-    previous_wants_init = os.environ.get("ARCHIVEBOX_WANTS_INIT")
-    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
-    try:
-        check_migrations(blocking=True, auto_apply=False)
-
-        for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
-            sys.stdout.write(f"    {migration_line}\n")
-    finally:
-        if previous_wants_init is None:
-            os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
-        else:
-            os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
-
-    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
-    print()
-    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
-
-    # from django.contrib.auth.models import User
-    #     call_command("createsuperuser", interactive=True)
-
-    print()
-    print("[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]")
-
-    from archivebox.core.models import Snapshot
-
-    snapshot_count = 0
-
-    if existing_index:
-        snapshot_count = Snapshot.objects.count()
-        print(f"    √ Loaded {snapshot_count} links from existing main index.")
-
-    print("    > Skipping orphan snapshot import during init.")
-    print()
-    print("    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
-    print("        archivebox update")
-
-    print("\n[green]----------------------------------------------------------------------[/green]")
-
-    from django.contrib.auth.models import User
-
-    config = get_config()
-    if (config.ADMIN_USERNAME and config.ADMIN_PASSWORD) and not User.objects.filter(
-        username=config.ADMIN_USERNAME,
-    ).exists():
-        print("[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]")
-        User.objects.create_superuser(username=config.ADMIN_USERNAME, password=config.ADMIN_PASSWORD)
-
-    if existing_index:
-        print("[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]")
-    else:
-        print(f"[green][√] Done. A new ArchiveBox collection was initialized ({snapshot_count} links).[/green]")
-
-    CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
-    CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
-
-    from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir
-
-    config = get_config()
-    config.TMP_DIR.mkdir(parents=True, exist_ok=True)
-    config.ABXPKG_LIB_DIR.mkdir(parents=True, exist_ok=True)
-    (config.ABXPKG_LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
-
-    working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
-    if working_tmp_dir:
-        working_tmp_dir.mkdir(parents=True, exist_ok=True)
-
-    working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
-    if working_lib_dir:
-        working_lib_dir.mkdir(parents=True, exist_ok=True)
-
-    if install:
-        from archivebox.cli.archivebox_install import install as install_method
-
-        install_method()
-
-    if Snapshot.objects.count() < 25:  # hide the hints for experienced users
-        print()
-        print("    [violet]Hint:[/violet] To view your archive index, run:")
-        print(
-            "        archivebox server  # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]",
-        )
-        print()
-        print("    To add new links, you can run:")
-        print("        archivebox add < ~/some/path/to/list_of_links.txt")
-        print()
-        print("    For more usage and examples, run:")
-        print("        archivebox help")
-
-
-@click.command()
-@click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
-@click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
-@click.option("--install", "--setup", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
@docstring(init.__doc__)
-def main(**kwargs) -> None:
-    init(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=init.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--force', # '-f',
+        action='store_true',
+        help='Ignore unrecognized files in current directory and initialize anyway',
+    )
+    parser.add_argument(
+        '--quick', '-q',
+        action='store_true',
+        help='Run any updates or migrations without rechecking all snapshot dirs',
+    )
+    parser.add_argument(
+        '--install', #'-s',
+        action='store_true',
+        help='Automatically install dependencies and extras used for archiving',
+    )
+    parser.add_argument(
+        '--setup', #'-s',
+        action='store_true',
+        help='DEPRECATED: equivalent to --install',
+    )
+    command = parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)

+    init(
+        force=command.force,
+        quick=command.quick,
+        install=command.install or command.setup,
+        out_dir=pwd or DATA_DIR,
+    )
+    

-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,145 +1,56 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox install'

-import os
-import shutil
+import sys
+import argparse
 from pathlib import Path
+from typing import Optional, List, IO

-import rich_click as click
-from rich import print
-
-from archivebox.misc.util import docstring, enforce_types
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
+from ..logging_util import SmartFormatter, reject_stdin
+from ..main import install


-def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:
-    """Expose the shared abxpkg lib dir at DATA_DIR/lib after install."""
-    link_path = data_dir / "lib"
-    target_config_path = abxpkg_lib_dir.expanduser().absolute()
-    target_path = abxpkg_lib_dir.expanduser().resolve()
-    from archivebox.config.paths import get_machine_type
-
-    scoped_data_lib_path = (link_path / get_machine_type()).absolute()
-
-    if target_config_path == scoped_data_lib_path:
-        if link_path.is_symlink() or link_path.is_file():
-            link_path.unlink()
-        target_config_path.mkdir(parents=True, exist_ok=True)
-        return None
-
-    if link_path.resolve(strict=False) == target_path:
-        return None
-
-    target_path.mkdir(parents=True, exist_ok=True)
-
-    if link_path.is_symlink():
-        if link_path.resolve(strict=False) == target_path:
-            return link_path
-        link_path.unlink()
-    elif link_path.exists():
-        shutil.rmtree(link_path)
-
-    relative_target = os.path.relpath(target_path, start=link_path.parent)
-    link_path.symlink_to(relative_target, target_is_directory=True)
-    return link_path
-
-
-@enforce_types
-def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
-    """Detect and install ArchiveBox dependencies by running the abx-dl install flow
-
-    Examples:
-        archivebox install                              # Install all dependencies
-        archivebox install wget curl                    # Install only wget and curl
-        archivebox install --binproviders=pip yt-dlp    # Install yt-dlp using only pip
-        archivebox install --binproviders=brew,apt      # Install all deps using only brew or apt
-    """
-
-    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
-    from archivebox.config import CONSTANTS
-    from archivebox.misc.logging import stderr
-    from archivebox.cli.archivebox_init import init
-
-    archive_dir = CONSTANTS.ARCHIVE_DIR
-
-    if dry_run:
-        print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")
-        return
-
-    if not (os.access(archive_dir, os.R_OK) and archive_dir.is_dir()):
-        init()  # must init full index because we need a db to store Binary entries in
-
-    # Show what we're installing
-    if binaries:
-        print(f"\n[green][+] Installing specific binaries: {', '.join(binaries)}[/green]")
-    else:
-        print("\n[green][+] Detecting and installing all ArchiveBox dependencies...[/green]")
-
-    if binproviders != "*":
-        print(f"[green][+] Using providers: {binproviders}[/green]")
-
-    if IS_ROOT:
-        EUID = os.geteuid()
-        print()
-        print(f"[yellow]:warning:  Running as UID=[blue]{EUID}[/blue].[/yellow]")
-        print(f"    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].")
-        print()
-
-    # Set up Django
-    from archivebox.config.django import setup_django
-
-    setup_django()
-
-    plugin_names = list(binaries)
-    if binproviders != "*":
-        plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
-
-    print("[+] Running installer via abx-dl bus...")
-    print()
-
-    from archivebox.services.runner import run_install
-
-    run_install(plugin_names=plugin_names or None)
-
-    from archivebox.config.common import get_config
-
-    config = get_config(include_machine=False)
-    ensure_data_dir_lib_symlink(CONSTANTS.DATA_DIR, config.ABXPKG_LIB_DIR)
-
-    print()
-
-    # Check for superuser
-    from django.contrib.auth import get_user_model
-
-    User = get_user_model()
-
-    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
-        stderr("\n[+] Don't forget to create a new admin user for the Web UI...", color="green")
-        stderr("    archivebox manage createsuperuser")
-
-    print()
-
-    # Show version to display full status including installed binaries
-    # Django is already loaded, so just import and call the function directly
-    from archivebox.cli.archivebox_version import version as show_version
-
-    show_version(quiet=False, binaries=binaries)
-
-
-@click.command()
-@click.argument("binaries", nargs=-1, type=str, required=False)
-@click.option(
-    "--binproviders",
-    "-p",
-    default="*",
-    help="Comma-separated list of providers to use (pip,npm,brew,apt,env,custom) or * for all",
-    show_default=True,
-)
-@click.option("--dry-run", "-d", is_flag=True, help="Show what would happen without actually running", default=False)
@docstring(install.__doc__)
-def main(**kwargs) -> None:
-    install(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=install.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--binproviders', '-p',
+        type=str,
+        help='Select binproviders to use DEFAULT=env,apt,brew,sys_pip,venv_pip,lib_pip,pipx,sys_npm,lib_npm,puppeteer,playwright (all)',
+        default=None,
+    )
+    parser.add_argument(
+        '--binaries', '-b',
+        type=str,
+        help='Select binaries to install DEFAULT=curl,wget,git,yt-dlp,chrome,single-file,readability-extractor,postlight-parser,... (all)',
+        default=None,
+    )
+    parser.add_argument(
+        '--dry-run', '-d',
+        action='store_true',
+        help='Show what would be installed without actually installing anything',
+        default=False,
+    )
+    command = parser.parse_args(args or ())   # noqa
+    reject_stdin(__command__, stdin)

+    install(
+        # force=command.force,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
+        binaries=command.binaries.split(',') if command.binaries else None,
+        binproviders=command.binproviders.split(',') if command.binproviders else None,
+        dry_run=command.dry_run,
+    )
+    

-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,22 +1,139 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
-__command__ = "archivebox list"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox list'

 import sys
+import argparse
+from pathlib import Path
+from typing import Optional, List, IO

-import rich_click as click
-
-from archivebox.cli.archivebox_snapshot import list_snapshots, snapshot_filter_options, snapshot_output_options
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
+from ..main import list_all
+from ..index import (
+    LINK_FILTERS,
+    get_indexed_folders,
+    get_archived_folders,
+    get_unarchived_folders,
+    get_present_folders,
+    get_valid_folders,
+    get_invalid_folders,
+    get_duplicate_folders,
+    get_orphaned_folders,
+    get_corrupted_folders,
+    get_unrecognized_folders,
+)
+from ..logging_util import SmartFormatter, reject_stdin, stderr


-@click.command()
-@snapshot_output_options
-@snapshot_filter_options(default_filter_type="substring")
-def main(**kwargs) -> None:
-    """List Snapshots."""
-    sys.exit(list_snapshots(**kwargs))
+@docstring(list_all.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=list_all.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--csv', #'-c',
+        type=str,
+        help="Print the output in CSV format with the given columns, e.g.: timestamp,url,extension",
+        default=None,
+    )
+    group.add_argument(
+        '--json', #'-j',
+        action='store_true',
+        help="Print the output in JSON format with all columns included",
+    )
+    group.add_argument(
+        '--html',
+        action='store_true',
+        help="Print the output in HTML format"
+    )
+    parser.add_argument(
+        '--with-headers',
+        action='store_true',
+        help='Include the headers in the output document' 
+    )
+    parser.add_argument(
+        '--sort', #'-s',
+        type=str,
+        help="List the links sorted using the given key, e.g. timestamp or updated",
+        default=None,
+    )
+    parser.add_argument(
+        '--before', #'-b',
+        type=float,
+        help="List only links bookmarked before (less than) the given timestamp",
+        default=None,
+    )
+    parser.add_argument(
+        '--after', #'-a',
+        type=float,
+        help="List only links bookmarked after (greater than or equal to) the given timestamp",
+        default=None,
+    )
+    parser.add_argument(
+        '--status',
+        type=str,
+        choices=('indexed', 'archived', 'unarchived', 'present', 'valid', 'invalid', 'duplicate', 'orphaned', 'corrupted', 'unrecognized'),
+        default='indexed',
+        help=(
+            'List only links or data directories that have the given status\n'
+            f'    indexed       {get_indexed_folders.__doc__} (the default)\n'
+            f'    archived      {get_archived_folders.__doc__}\n'
+            f'    unarchived    {get_unarchived_folders.__doc__}\n'
+            '\n'
+            f'    present       {get_present_folders.__doc__}\n'
+            f'    valid         {get_valid_folders.__doc__}\n'
+            f'    invalid       {get_invalid_folders.__doc__}\n'
+            '\n'
+            f'    duplicate     {get_duplicate_folders.__doc__}\n'
+            f'    orphaned      {get_orphaned_folders.__doc__}\n'
+            f'    corrupted     {get_corrupted_folders.__doc__}\n'
+            f'    unrecognized  {get_unrecognized_folders.__doc__}\n'
+        )
+    )
+    parser.add_argument(
+        '--filter-type', '-t',
+        type=str,
+        choices=(*LINK_FILTERS.keys(), 'search'),
+        default='exact',
+        help='Type of pattern matching to use when filtering URLs',
+    )
+    parser.add_argument(
+        'filter_patterns',
+        nargs='*',
+        type=str,
+        default=None,
+        help='List only URLs matching these filter patterns'
+    )
+    command = parser.parse_args(args or ())
+    reject_stdin(stdin)

+    if command.with_headers and not (command.json or command.html or command.csv):
+        stderr(
+            '[X] --with-headers can only be used with --json, --html or --csv options\n',
+            color='red',
+        )
+        raise SystemExit(2)

-if __name__ == "__main__":
-    main()
+    matching_folders = list_all(
+        filter_patterns=command.filter_patterns,
+        filter_type=command.filter_type,
+        status=command.status,
+        after=command.after,
+        before=command.before,
+        sort=command.sort,
+        csv=command.csv,
+        json=command.json,
+        html=command.html,
+        with_headers=command.with_headers,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
+    )
+    raise SystemExit(not matching_folders)
+
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,102 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox machine <action> [--filters]
-
-Manage Machine records (system-managed, mostly read-only).
-
-Machine records track the host machines where ArchiveBox runs.
-They are created automatically by the system and are primarily for debugging.
-
-Actions:
-    list    - List Machines as JSONL (with optional filters)
-
-Examples:
-    # List all machines
-    archivebox machine list
-
-    # List machines by hostname
-    archivebox machine list --hostname__icontains=myserver
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox machine"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_util import apply_filters
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_machines(
-    hostname__icontains: str | None = None,
-    os_platform: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Machines as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Machine
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Machine.objects.all().order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "hostname__icontains": hostname__icontains,
-        "os_platform": os_platform,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for machine in queryset:
-        if is_tty:
-            rprint(f"[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}")
-        else:
-            write_record(machine.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} machines[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Machine records (read-only, system-managed)."""
-    pass
-
-
-@main.command("list")
-@click.option("--hostname__icontains", help="Filter by hostname contains")
-@click.option("--os-platform", help="Filter by OS platform")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(hostname__icontains: str | None, os_platform: str | None, limit: int | None):
-    """List Machines as JSONL."""
-    sys.exit(
-        list_machines(
-            hostname__icontains=hostname__icontains,
-            os_platform=os_platform,
-            limit=limit,
-        ),
-    )
-
-
-if __name__ == "__main__":
-    main()
@@ -1,35 +1,24 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox manage'

-import rich_click as click
-from archivebox.misc.util import docstring, enforce_types
+import sys
+from pathlib import Path
+from typing import Optional, List, IO
+
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
+from ..main import manage


-@enforce_types
-def manage(args: list[str] | None = None) -> None:
-    """Run an ArchiveBox Django management command"""
-
-    from archivebox.config.common import get_config
-    from archivebox.misc.logging import stderr
-
-    config = get_config()
-    if (args and "createsuperuser" in args) and (config.IN_DOCKER and not config.IS_TTY):
-        stderr("[!] Warning: you need to pass -it to use interactive commands in docker", color="lightyellow")
-        stderr("    docker run -it archivebox manage {}".format(" ".join(args or ["..."])), color="lightyellow")
-        stderr("")
-
-    from django.core.management import execute_from_command_line
-
-    execute_from_command_line(["manage.py", *(args or ["help"])])
-
-
-@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
-@click.argument("args", nargs=-1)
@docstring(manage.__doc__)
-def main(args: list[str] | None = None) -> None:
-    manage(args=args)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    manage(
+        args=args,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
+    )


-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,49 +0,0 @@
-#!/usr/bin/env python3
-"""
-archivebox mcp
-
-Start the Model Context Protocol (MCP) server in stdio mode.
-Exposes all ArchiveBox CLI commands as MCP tools for AI agents.
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox mcp"
-
-import rich_click as click
-
-from archivebox.misc.util import docstring, enforce_types
-
-
-@enforce_types
-def mcp():
-    """
-    Start the MCP server in stdio mode for AI agent control.
-
-    The MCP (Model Context Protocol) server exposes all ArchiveBox CLI commands
-    as tools that AI agents can discover and execute. It communicates via JSON-RPC
-    2.0 over stdin/stdout.
-
-    Example usage with an MCP client:
-        archivebox mcp < requests.jsonl > responses.jsonl
-
-    Or interactively:
-        archivebox mcp
-        {"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}
-        {"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}
-    """
-
-    from archivebox.mcp.server import run_mcp_server
-
-    # Run the stdio server (blocks until stdin closes)
-    run_mcp_server()
-
-
-@click.command()
-@docstring(mcp.__doc__)
-def main(**kwargs):
-    """Start the MCP server in stdio mode"""
-    mcp()
-
-
-if __name__ == "__main__":
-    main()
@@ -1,27 +1,73 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
-__command__ = "archivebox oneshot"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox oneshot'
+
+import sys
+import argparse

-import subprocess
 from pathlib import Path
+from typing import List, Optional, IO

-import rich_click as click
-
-from archivebox.config import CONSTANTS
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
+from ..logging_util import SmartFormatter, accept_stdin, stderr
+from ..main import oneshot


-@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
-@click.argument("args", nargs=-1)
-def main(args: tuple[str, ...] = ()) -> None:
-    """Download URLs using abx-dl"""
-    cwd = Path.cwd()
-    if any((path / CONSTANTS.SQL_INDEX_FILENAME).exists() for path in (cwd, *cwd.parents)):
-        raise click.ClickException(
-            "Refusing to run `archivebox oneshot` inside an ArchiveBox DATA_DIR. Use `archivebox add` here, or run oneshot from another directory.",
+@docstring(oneshot.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=oneshot.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        'url',
+        type=str,
+        default=None,
+        help=(
+            'URLs or paths to archive e.g.:\n'
+            '    https://getpocket.com/users/USERNAME/feed/all\n'
+            '    https://example.com/some/rss/feed.xml\n'
+            '    https://example.com\n'
+            '    ~/Downloads/firefox_bookmarks_export.html\n'
+            '    ~/Desktop/sites_list.csv\n'
        )
-    raise SystemExit(subprocess.run(["abx-dl", *args]).returncode)
+    )
+    parser.add_argument(
+        "--extract",
+        type=str,
+        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
+              This does not take precedence over the configuration",
+        default=""
+    )
+    parser.add_argument(
+        '--out-dir',
+        type=str,
+        default=DATA_DIR,
+        help= "Path to save the single archive folder to, e.g. ./example.com_archive"
+    )
+    command = parser.parse_args(args or ())
+    stdin_url = None
+    url = command.url
+    if not url:
+        stdin_url = accept_stdin(stdin)
+
+    if (stdin_url and url) or (not stdin and not url):
+        stderr(
+            '[X] You must pass a URL/path to add via stdin or CLI arguments.\n',
+            color='red',
+        )
+        raise SystemExit(2)
+    
+    oneshot(
+        url=stdin_url or url,
+        out_dir=Path(command.out_dir).resolve(),
+        extractors=command.extract,
+    )


-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,579 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox persona <action> [args...] [--filters]
-
-Manage Persona records (browser profiles for archiving).
-
-Actions:
-    create  - Create Personas
-    list    - List Personas as JSONL (with optional filters)
-    update  - Update Personas from stdin JSONL
-    delete  - Delete Personas from stdin JSONL
-
-Examples:
-    # Create a new persona
-    archivebox persona create work
-    archivebox persona create --import=chrome personal
-    archivebox persona create --import=edge work
-
-    # List all personas
-    archivebox persona list
-
-    # Delete a persona
-    archivebox persona list --name=old | archivebox persona delete --yes
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox persona"
-
-import os
-import sys
-import shutil
-import platform
-from pathlib import Path
-from collections.abc import Iterable
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_util import apply_filters
-from archivebox.personas import importers as persona_importers
-
-
-# =============================================================================
-# Browser Profile Locations
-# =============================================================================
-
-
-def get_chrome_user_data_dir() -> Path | None:
-    """Get the default Chrome user data directory for the current platform."""
-    system = platform.system()
-    home = Path.home()
-
-    if system == "Darwin":  # macOS
-        candidates = [
-            home / "Library" / "Application Support" / "Google" / "Chrome",
-            home / "Library" / "Application Support" / "Chromium",
-        ]
-    elif system == "Linux":
-        candidates = [
-            home / ".config" / "google-chrome",
-            home / ".config" / "chromium",
-            home / ".config" / "chrome",
-            home / "snap" / "chromium" / "common" / "chromium",
-        ]
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = [
-            local_app_data / "Google" / "Chrome" / "User Data",
-            local_app_data / "Chromium" / "User Data",
-        ]
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate.exists() and (candidate / "Default").exists():
-            return candidate
-
-    return None
-
-
-def get_brave_user_data_dir() -> Path | None:
-    """Get the default Brave user data directory for the current platform."""
-    system = platform.system()
-    home = Path.home()
-
-    if system == "Darwin":
-        candidates = [
-            home / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser",
-        ]
-    elif system == "Linux":
-        candidates = [
-            home / ".config" / "BraveSoftware" / "Brave-Browser",
-        ]
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = [
-            local_app_data / "BraveSoftware" / "Brave-Browser" / "User Data",
-        ]
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate.exists() and (candidate / "Default").exists():
-            return candidate
-
-    return None
-
-
-def get_edge_user_data_dir() -> Path | None:
-    """Get the default Edge user data directory for the current platform."""
-    system = platform.system()
-    home = Path.home()
-
-    if system == "Darwin":
-        candidates = [
-            home / "Library" / "Application Support" / "Microsoft Edge",
-        ]
-    elif system == "Linux":
-        candidates = [
-            home / ".config" / "microsoft-edge",
-            home / ".config" / "microsoft-edge-beta",
-            home / ".config" / "microsoft-edge-dev",
-        ]
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = [
-            local_app_data / "Microsoft" / "Edge" / "User Data",
-        ]
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate.exists() and (candidate / "Default").exists():
-            return candidate
-
-    return None
-
-
-BROWSER_PROFILE_FINDERS = {
-    "chrome": get_chrome_user_data_dir,
-    "chromium": get_chrome_user_data_dir,  # Same locations
-    "brave": get_brave_user_data_dir,
-    "edge": get_edge_user_data_dir,
-}
-
-CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"}
-
-
-# =============================================================================
-# Cookie Extraction via CDP
-# =============================================================================
-
-# =============================================================================
-# Validation Helpers
-# =============================================================================
-
-
-def validate_persona_name(name: str) -> tuple[bool, str]:
-    """
-    Validate persona name to prevent path traversal attacks.
-
-    Returns:
-        (is_valid, error_message): tuple indicating if name is valid
-    """
-    if not name or not name.strip():
-        return False, "Persona name cannot be empty"
-
-    # Check for path separators
-    if "/" in name or "\\" in name:
-        return False, "Persona name cannot contain path separators (/ or \\)"
-
-    # Check for parent directory references
-    if ".." in name:
-        return False, "Persona name cannot contain parent directory references (..)"
-
-    # Check for hidden files/directories
-    if name.startswith("."):
-        return False, "Persona name cannot start with a dot (.)"
-
-    # Ensure name doesn't contain null bytes or other dangerous chars
-    if "\x00" in name or "\n" in name or "\r" in name:
-        return False, "Persona name contains invalid characters"
-
-    return True, ""
-
-
-def ensure_path_within_personas_dir(persona_path: Path) -> bool:
-    """
-    Verify that a persona path is within PERSONAS_DIR.
-
-    This is a safety check to prevent path traversal attacks where
-    a malicious persona name could cause operations on paths outside
-    the expected PERSONAS_DIR.
-
-    Returns:
-        True if path is safe, False otherwise
-    """
-    from archivebox.config.constants import CONSTANTS
-
-    try:
-        # Resolve both paths to absolute paths
-        personas_dir = CONSTANTS.PERSONAS_DIR.resolve()
-        resolved_path = persona_path.resolve()
-
-        # Check if resolved_path is a child of personas_dir
-        return resolved_path.is_relative_to(personas_dir)
-    except (ValueError, RuntimeError):
-        return False
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_personas(
-    names: Iterable[str],
-    import_from: str | None = None,
-    profile: str | None = None,
-) -> int:
-    """
-    Create Personas from names.
-
-    If --import is specified, copy the browser profile to the persona directory
-    and extract cookies.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.personas.models import Persona
-
-    is_tty = sys.stdout.isatty()
-    name_list = list(names) if names else []
-
-    if not name_list:
-        rprint("[yellow]No persona names provided. Pass names as arguments.[/yellow]", file=sys.stderr)
-        return 1
-
-    # Validate import source if specified
-    source_profile_dir = None
-    if import_from:
-        import_from = import_from.lower()
-        if import_from not in BROWSER_PROFILE_FINDERS:
-            rprint(f"[red]Unknown browser: {import_from}[/red]", file=sys.stderr)
-            rprint(f"[dim]Supported browsers: {', '.join(BROWSER_PROFILE_FINDERS.keys())}[/dim]", file=sys.stderr)
-            return 1
-
-        source_profile_dir = BROWSER_PROFILE_FINDERS[import_from]()
-        if not source_profile_dir:
-            rprint(f"[red]Could not find {import_from} profile directory[/red]", file=sys.stderr)
-            return 1
-
-        rprint(f"[dim]Found {import_from} profile: {source_profile_dir}[/dim]", file=sys.stderr)
-
-        if profile is None and (source_profile_dir / "Default").exists():
-            profile = "Default"
-
-    created_count = 0
-    for name in name_list:
-        name = name.strip()
-        if not name:
-            continue
-
-        # Validate persona name to prevent path traversal
-        is_valid, error_msg = persona_importers.validate_persona_name(name)
-        if not is_valid:
-            rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
-            continue
-
-        persona, created = Persona.objects.get_or_create(name=name)
-
-        if created:
-            persona.ensure_dirs()
-            created_count += 1
-            rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
-        else:
-            rprint(f"[dim]Persona already exists: {name}[/dim]", file=sys.stderr)
-
-        cookies_file = Path(persona.path) / "cookies.txt"
-
-        # Import browser profile if requested
-        if import_from in CHROMIUM_BROWSERS and source_profile_dir is not None:
-            try:
-                import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile)
-                import_result = persona_importers.import_persona_from_source(
-                    persona,
-                    import_source,
-                    copy_profile=True,
-                    import_cookies=True,
-                    capture_storage=False,
-                )
-            except Exception as e:
-                rprint(f"[red]Failed to import browser profile: {e}[/red]", file=sys.stderr)
-                return 1
-
-            if import_result.profile_copied:
-                rprint("[green]Copied browser profile to persona[/green]", file=sys.stderr)
-            if import_result.cookies_imported:
-                rprint(f"[green]Extracted cookies to {cookies_file}[/green]", file=sys.stderr)
-            elif not import_result.profile_copied:
-                rprint("[yellow]Could not import cookies automatically.[/yellow]", file=sys.stderr)
-
-            for warning in import_result.warnings:
-                rprint(f"[yellow]{warning}[/yellow]", file=sys.stderr)
-
-        if not is_tty:
-            write_record(
-                {
-                    "id": str(persona.id),
-                    "name": persona.name,
-                    "path": str(persona.path),
-                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
-                    "COOKIES_FILE": persona.COOKIES_FILE,
-                },
-            )
-
-    rprint(f"[green]Created {created_count} new persona(s)[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_personas(
-    name: str | None = None,
-    name__icontains: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Personas as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.personas.models import Persona
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Persona.objects.all().order_by("name")
-
-    # Apply filters
-    filter_kwargs = {
-        "name": name,
-        "name__icontains": name__icontains,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for persona in queryset:
-        cookies_status = "[green]✓[/green]" if persona.COOKIES_FILE else "[dim]✗[/dim]"
-        chrome_status = "[green]✓[/green]" if Path(persona.CHROME_USER_DATA_DIR).exists() else "[dim]✗[/dim]"
-
-        if is_tty:
-            rprint(f"[cyan]{persona.name:20}[/cyan] cookies:{cookies_status} chrome:{chrome_status} [dim]{persona.path}[/dim]")
-        else:
-            write_record(
-                {
-                    "id": str(persona.id),
-                    "name": persona.name,
-                    "path": str(persona.path),
-                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
-                    "COOKIES_FILE": persona.COOKIES_FILE,
-                },
-            )
-        count += 1
-
-    rprint(f"[dim]Listed {count} persona(s)[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_personas(name: str | None = None) -> int:
-    """
-    Update Personas from stdin JSONL.
-
-    Reads Persona records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.personas.models import Persona
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        persona_id = record.get("id")
-        old_name = record.get("name")
-
-        if not persona_id and not old_name:
-            continue
-
-        try:
-            if persona_id:
-                persona = Persona.objects.get(id=persona_id)
-            else:
-                persona = Persona.objects.get(name=old_name)
-
-            # Apply updates from CLI flags
-            if name:
-                # Validate new name to prevent path traversal
-                is_valid, error_msg = persona_importers.validate_persona_name(name)
-                if not is_valid:
-                    rprint(f'[red]Invalid new persona name "{name}": {error_msg}[/red]', file=sys.stderr)
-                    continue
-
-                # Rename the persona directory too
-                old_path = persona.path
-                persona.name = name
-                new_path = persona.path
-
-                if old_path.exists() and old_path != new_path:
-                    shutil.move(str(old_path), str(new_path))
-
-                persona.save()
-
-            updated_count += 1
-
-            if not is_tty:
-                write_record(
-                    {
-                        "id": str(persona.id),
-                        "name": persona.name,
-                        "path": str(persona.path),
-                    },
-                )
-
-        except Persona.DoesNotExist:
-            rprint(f"[yellow]Persona not found: {persona_id or old_name}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} persona(s)[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_personas(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Personas from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.personas.models import Persona
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    # Collect persona IDs or names
-    persona_ids = []
-    persona_names = []
-    for r in records:
-        if r.get("id"):
-            persona_ids.append(r["id"])
-        elif r.get("name"):
-            persona_names.append(r["name"])
-
-    if not persona_ids and not persona_names:
-        rprint("[yellow]No valid persona IDs or names in input[/yellow]", file=sys.stderr)
-        return 1
-
-    from django.db.models import Q
-
-    query = Q()
-    if persona_ids:
-        query |= Q(id__in=persona_ids)
-    if persona_names:
-        query |= Q(name__in=persona_names)
-
-    personas = Persona.objects.filter(query)
-    count = personas.count()
-
-    if count == 0:
-        rprint("[yellow]No matching personas found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} persona(s) (dry run)[/yellow]", file=sys.stderr)
-        for persona in personas:
-            rprint(f"  {persona.name} ({persona.path})", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Delete persona directories and database records
-    deleted_count = 0
-    for persona in personas:
-        persona_path = persona.path
-
-        # Safety check: ensure path is within PERSONAS_DIR before deletion
-        if not ensure_path_within_personas_dir(persona_path):
-            rprint(f'[red]Security error: persona path "{persona_path}" is outside PERSONAS_DIR. Skipping deletion.[/red]', file=sys.stderr)
-            continue
-
-        if persona_path.exists():
-            shutil.rmtree(persona_path)
-        persona.delete()
-        deleted_count += 1
-
-    rprint(f"[green]Deleted {deleted_count} persona(s)[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Persona records (browser profiles)."""
-    pass
-
-
-@main.command("create")
-@click.argument("names", nargs=-1)
-@click.option("--import", "import_from", help="Import profile from browser (chrome, chromium, brave, edge)")
-@click.option("--profile", help="Profile directory name under the user data dir (e.g. Default, Profile 1)")
-def create_cmd(names: tuple, import_from: str | None, profile: str | None):
-    """Create Personas, optionally importing from a browser profile."""
-    sys.exit(create_personas(names, import_from=import_from, profile=profile))
-
-
-@main.command("list")
-@click.option("--name", help="Filter by exact name")
-@click.option("--name__icontains", help="Filter by name contains")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
-    """List Personas as JSONL."""
-    sys.exit(list_personas(name=name, name__icontains=name__icontains, limit=limit))
-
-
-@main.command("update")
-@click.option("--name", "-n", help="Set new name")
-def update_cmd(name: str | None):
-    """Update Personas from stdin JSONL."""
-    sys.exit(update_personas(name=name))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Personas from stdin JSONL."""
-    sys.exit(delete_personas(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,233 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-
-
-import rich_click as click
-
-from archivebox.misc.util import docstring, enforce_types
-
-
-EVENT_FLOW_DIAGRAM = """
-┌─────────────────────────────────────────────────────────────────────────────┐
-│                           ArchiveBox / abx-dl Flow                         │
-├─────────────────────────────────────────────────────────────────────────────┤
-│                                                                             │
-│  InstallEvent                                                               │
-│    └─ config.json > required_binaries                                       │
-│         └─ BinaryRequestEvent                                               │
-│              └─ abxpkg BinaryService builtin providers                      │
-│                   └─ BinaryEvent                                            │
-│                        └─ BinaryCacheService / project cache backend        │
-│                                                                             │
-│  CrawlEvent                                                                 │
-│    └─ CrawlSetupEvent                                                       │
-│         └─ on_CrawlSetup__*                                                 │
-│                                                                             │
-│  CrawlStartEvent                                                            │
-│    └─ SnapshotEvent                                                         │
-│         └─ on_Snapshot__*                                                   │
-│              └─ ArchiveResult / Snapshot / Tag                              │
-│                                                                             │
-│  SnapshotCleanupEvent  -> internal cleanup, no direct hook family           │
-│  CrawlCleanupEvent     -> internal cleanup, no direct hook family           │
-│                                                                             │
-│  ArchiveBox projects bus events into the DB; it no longer drives plugin     │
-│  execution through the old queued model executor.                           │
-└─────────────────────────────────────────────────────────────────────────────┘
-"""
-
-
-@enforce_types
-def pluginmap(
-    show_disabled: bool = False,
-    event: str | None = None,
-    quiet: bool = False,
-) -> dict:
-    """
-    Show the current abx-dl event phases and their associated plugin hooks.
-
-    This command reflects the new bus-driven runtime, not the legacy ArchiveBox
-    state-machine executor. Event names are normalized to hook prefixes by
-    stripping a trailing `Event`, then ArchiveBox checks whether any matching
-    `on_{EventFamily}__*` scripts actually exist.
-    """
-    from rich.console import Console
-    from rich.table import Table
-    from rich.panel import Panel
-    from rich import box
-
-    from archivebox.plugins.hooks import (
-        discover_hooks,
-        is_background_hook,
-        normalize_hook_event_name,
-    )
-    from archivebox.plugins.discovery import (
-        BUILTIN_PLUGINS_DIR,
-        USER_PLUGINS_DIR,
-    )
-
-    console = Console()
-    prnt = console.print
-
-    event_phases = {
-        "InstallEvent": {
-            "description": "Pre-run dependency phase. Enabled plugins emit BinaryRequest events from config.json required_binaries.",
-            "emits": ["BinaryRequestEvent", "BinaryEvent", "ProcessEvent"],
-        },
-        "BinaryRequestEvent": {
-            "description": "Binary resolution phase. abxpkg BinaryService resolves or installs requested binaries using built-in providers.",
-            "emits": ["BinaryEvent", "ProcessEvent"],
-            "direct_hooks": False,
-        },
-        "BinaryEvent": {
-            "description": "Resolved binary metadata event. ArchiveBoxBinaryService projects it into the ArchiveBox DB binary cache.",
-            "emits": [],
-            "direct_hooks": False,
-        },
-        "CrawlEvent": {
-            "description": "Root crawl lifecycle event emitted by the runner.",
-            "emits": ["CrawlSetupEvent", "CrawlStartEvent", "CrawlCleanupEvent", "CrawlCompletedEvent"],
-        },
-        "CrawlSetupEvent": {
-            "description": "Crawl-scoped setup phase. on_CrawlSetup hooks launch/configure shared daemons and runtime state.",
-            "emits": ["ProcessEvent"],
-        },
-        "SnapshotEvent": {
-            "description": "Per-snapshot extraction phase. on_Snapshot hooks emit ArchiveResult, Snapshot, and Tag records.",
-            "emits": ["ArchiveResultEvent", "SnapshotEvent", "TagEvent", "ProcessEvent"],
-        },
-        "SnapshotCleanupEvent": {
-            "description": "Internal snapshot cleanup phase.",
-            "emits": ["ProcessKillEvent"],
-        },
-        "CrawlCleanupEvent": {
-            "description": "Internal crawl cleanup phase.",
-            "emits": ["ProcessKillEvent"],
-        },
-    }
-
-    if event:
-        requested = str(event).strip()
-        if requested in event_phases:
-            event_phases = {requested: event_phases[requested]}
-        else:
-            normalized_requested = normalize_hook_event_name(requested)
-            matched_name = next((name for name in event_phases if normalize_hook_event_name(name) == normalized_requested), None)
-            if matched_name is None:
-                prnt(f'[red]Error: Unknown event "{requested}". Available: {", ".join(event_phases.keys())}[/red]')
-                return {}
-            event_phases = {matched_name: event_phases[matched_name]}
-
-    result = {
-        "events": {},
-        "plugins_dir": str(BUILTIN_PLUGINS_DIR),
-        "user_plugins_dir": str(USER_PLUGINS_DIR),
-    }
-
-    if not quiet:
-        prnt()
-        prnt("[bold cyan]ArchiveBox Plugin Map[/bold cyan]")
-        prnt(f"[dim]Built-in plugins: {BUILTIN_PLUGINS_DIR}[/dim]")
-        prnt(f"[dim]User plugins: {USER_PLUGINS_DIR}[/dim]")
-        prnt()
-        prnt(
-            Panel(
-                EVENT_FLOW_DIAGRAM,
-                title="[bold green]Event Flow[/bold green]",
-                border_style="green",
-                expand=False,
-            ),
-        )
-        prnt()
-
-    for event_name, info in event_phases.items():
-        hook_event = normalize_hook_event_name(event_name)
-        has_direct_hooks = info.get("direct_hooks", True)
-        hooks = discover_hooks(event_name, filter_disabled=not show_disabled) if has_direct_hooks else []
-
-        hook_infos = []
-        for hook_path in hooks:
-            plugin_name = hook_path.parent.name
-            hook_infos.append(
-                {
-                    "path": str(hook_path),
-                    "name": hook_path.name,
-                    "plugin": plugin_name,
-                    "is_background": is_background_hook(hook_path.name),
-                    "extension": hook_path.suffix,
-                },
-            )
-
-        result["events"][event_name] = {
-            "description": info["description"],
-            "hook_event": hook_event,
-            "emits": info["emits"],
-            "hooks": hook_infos,
-            "hook_count": len(hook_infos),
-        }
-
-        if quiet:
-            continue
-
-        title_suffix = f" -> on_{hook_event}__*" if hook_infos else ""
-        table = Table(
-            title=f"[bold yellow]{event_name}[/bold yellow]{title_suffix} ({len(hooks)} hooks)",
-            box=box.ROUNDED,
-            show_header=True,
-            header_style="bold magenta",
-        )
-        table.add_column("Plugin", style="cyan", width=20)
-        table.add_column("Hook Name", style="green")
-        table.add_column("BG", justify="center", width=4)
-        table.add_column("Type", justify="center", width=5)
-
-        if hook_infos:
-            for hook in sorted(hook_infos, key=lambda h: h["name"]):
-                bg_marker = "[yellow]bg[/yellow]" if hook["is_background"] else ""
-                table.add_row(
-                    hook["plugin"],
-                    hook["name"],
-                    bg_marker,
-                    hook["extension"].lstrip("."),
-                )
-        else:
-            table.add_row("[dim]-[/dim]", "[dim]No direct hooks[/dim]", "", "")
-
-        prnt(table)
-        prnt(f"[dim]{info['description']}[/dim]")
-        if info["emits"]:
-            prnt(f"[dim]Emits: {', '.join(info['emits'])}[/dim]")
-        if not hook_infos and has_direct_hooks:
-            prnt(f"[dim]No direct on_{hook_event}__* scripts are currently defined for this event family.[/dim]")
-        elif not has_direct_hooks:
-            prnt("[dim]No direct plugin hook family. This event is handled by services.[/dim]")
-        prnt()
-
-    if not quiet:
-        total_hooks = sum(event_info["hook_count"] for event_info in result["events"].values())
-        prnt(f"[bold]Total hooks discovered: {total_hooks}[/bold]")
-        prnt()
-        prnt("[dim]Hook naming convention: on_{EventFamily}__{XX}_{description}[.bg].{ext}[/dim]")
-        prnt("[dim]Event names are normalized with a simple `Event` suffix strip before hook discovery.[/dim]")
-        prnt("[dim]If no `on_{EventFamily}__*` scripts exist, the event is shown as having no direct hooks.[/dim]")
-        prnt()
-
-    return result
-
-
-@click.command()
-@click.option("--show-disabled", "-a", is_flag=True, help="Show hooks from disabled plugins too")
-@click.option("--event", "-e", type=str, default=None, help="Filter to specific event (e.g. InstallEvent, SnapshotEvent)")
-@click.option("--quiet", "-q", is_flag=True, help="Output JSON only, no tables")
-@docstring(pluginmap.__doc__)
-def main(**kwargs):
-    import json
-
-    result = pluginmap(**kwargs)
-    if kwargs.get("quiet"):
-        print(json.dumps(result, indent=2))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,110 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox process <action> [--filters]
-
-Manage Process records (system-managed, mostly read-only).
-
-Process records track executions of binaries during extraction.
-They are created automatically by the system and are primarily for debugging.
-
-Actions:
-    list    - List Processes as JSONL (with optional filters)
-
-Examples:
-    # List all processes
-    archivebox process list
-
-    # List processes by binary
-    archivebox process list --binary-name=chrome
-
-    # List recent processes
-    archivebox process list --limit=10
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox process"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_util import apply_filters
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_processes(
-    binary_name: str | None = None,
-    machine_id: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Processes as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Process
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Process.objects.all().select_related("binary", "machine").order_by("-started_at", "-created_at")
-
-    # Apply filters
-    filter_kwargs = {}
-    if binary_name:
-        filter_kwargs["binary__name"] = binary_name
-    if machine_id:
-        filter_kwargs["machine_id"] = machine_id
-
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for process in queryset:
-        if is_tty:
-            binary_name_str = process.binary.name if process.binary else "unknown"
-            exit_code = process.exit_code if process.exit_code is not None else "?"
-            status_color = "green" if process.exit_code == 0 else "red" if process.exit_code else "yellow"
-            rprint(f"[{status_color}]exit={exit_code:3}[/{status_color}] [cyan]{binary_name_str:15}[/cyan] [dim]{process.id}[/dim]")
-        else:
-            write_record(process.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} processes[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Process records (read-only, system-managed)."""
-    pass
-
-
-@main.command("list")
-@click.option("--binary-name", "-b", help="Filter by binary name")
-@click.option("--machine-id", "-m", help="Filter by machine ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(binary_name: str | None, machine_id: str | None, limit: int | None):
-    """List Processes as JSONL."""
-    sys.exit(
-        list_processes(
-            binary_name=binary_name,
-            machine_id=machine_id,
-            limit=limit,
-        ),
-    )
-
-
-if __name__ == "__main__":
-    main()
@@ -1,158 +1,82 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
-__command__ = "archivebox remove"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox remove'

-import time
+import sys
+import argparse
 from pathlib import Path
-from collections.abc import Iterable
+from typing import Optional, List, IO

-import rich_click as click
-
-from django.db import OperationalError
-from django.db.models import QuerySet
-
-from archivebox.config import CONSTANTS
-from archivebox.config.django import setup_django
-from archivebox.misc.util import enforce_types, docstring
-from archivebox.misc.checks import check_data_folder
-from archivebox.misc.logging_util import (
-    log_list_started,
-    log_list_finished,
-    log_removal_started,
-    log_removal_finished,
-    TimedProgress,
-)
-from archivebox.cli.archivebox_snapshot import snapshot_filter_options
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
+from ..logging_util import SmartFormatter, accept_stdin
+from ..main import remove


-@enforce_types
-def remove(
-    filter_patterns: Iterable[str] = (),
-    filter_type: str = "exact",
-    snapshots: QuerySet | None = None,
-    after: float | None = None,
-    before: float | None = None,
-    yes: bool = False,
-    delete: bool = False,
-    out_dir: Path = CONSTANTS.DATA_DIR,
-    timeout: float | None = None,
-    **kwargs,
-) -> dict[str, object]:
-    """Remove the specified URLs from the archive"""
-
-    setup_django()
-    check_data_folder()
-    timeout = float(timeout) if timeout is not None else None
-
-    from archivebox.core.models import Snapshot
-
-    filter_kwargs = {
-        **kwargs,
-        "filter_patterns": filter_patterns,
-        "filter_type": filter_type,
-        "after": after,
-        "before": before,
-    }
-    pattern_list = list(filter_patterns)
-
-    log_list_started(pattern_list or None, filter_type)
-    timer = TimedProgress(360, prefix="      ")
-    try:
-        if snapshots is None:
-            snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
-        # Freeze the target set up-front so a concurrent daemon writing new
-        # snapshots can't extend the deletion under us, and so the cursor isn't
-        # held open across the per-row deletes below.
-        snapshot_pks = list(snapshots.values_list("pk", flat=True))
-    finally:
-        timer.end()
-
-    if not snapshot_pks:
-        log_removal_finished(0, 0)
-        raise SystemExit(1)
-
-    if not yes:
-        log_list_finished(snapshots)
-        log_removal_started(snapshots, yes=False)
-
-    from archivebox.search.query import flush_search_index
-
-    started_at = time.monotonic()
-    deadline = started_at + timeout if timeout is not None else None
-
-    # Search-index flush touches a separate backend (FTS / sonic), not the
-    # main index.sqlite3 writer lock, so it's safe to do once up front.
-    flush_search_index(snapshots=snapshots)
-
-    # Delete one snapshot at a time. Each ``.delete()`` is its own short
-    # Django-atomic block, so the writer lock is released between rows and
-    # an in-flight daemon transaction can interleave instead of deadlocking.
-    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
-    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
-    # rmtree doesn't hold the lock either.
-    #
-    # The SQLite retry wrapper in core/sqlite_backend/base.py re-raises lock
-    # errors when called inside an atomic block (because it can't safely
-    # release+reacquire a transaction), so we wrap each row's delete in our
-    # own retry loop at this outer (non-atomic) level. Each attempt is a
-    # fresh atomic; an exception cleanly rolls it back before we sleep.
-    retry_interval = 1.0
-    deleted_snapshot_pks = []
-    timed_out = False
-    timeout_error = ""
-    for index, pk in enumerate(snapshot_pks):
-        if deadline is not None and time.monotonic() >= deadline:
-            timed_out = True
-            timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
-            break
-        while True:
-            try:
-                deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
-                if deleted_count:
-                    deleted_snapshot_pks.append(pk)
-                break
-            except OperationalError as err:
-                if "database is locked" not in str(err):
-                    raise
-                remaining_time = deadline - time.monotonic() if deadline is not None else None
-                if remaining_time is not None and remaining_time <= 0:
-                    timed_out = True
-                    timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock."
-                    break
-                time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval)
-        if timed_out:
-            break
-
-    all_snapshots = Snapshot.objects.all()
-    remaining_count = all_snapshots.count()
-    deleted_snapshot_id_set = set(deleted_snapshot_pks)
-    remaining_snapshot_pks = [snapshot_id for snapshot_id in snapshot_pks if snapshot_id not in deleted_snapshot_id_set]
-    log_removal_finished(remaining_count, len(deleted_snapshot_pks))
-
-    return {
-        "removed_count": len(deleted_snapshot_pks),
-        "removed_snapshot_ids": [str(snapshot_id) for snapshot_id in deleted_snapshot_pks],
-        "not_removed_count": len(remaining_snapshot_pks),
-        "not_removed_snapshot_ids": [str(snapshot_id) for snapshot_id in remaining_snapshot_pks],
-        "success": not timed_out,
-        "error": timeout_error,
-        "timeout": timeout,
-    }
-
-
-@click.command()
-@click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--delete", is_flag=True, help="Compatibility no-op; archived content is deleted automatically")
-@click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
-@snapshot_filter_options(default_filter_type="exact")
@docstring(remove.__doc__)
-def main(**kwargs):
-    """Remove the specified URLs from the archive"""
-    result = remove(**kwargs)
-    if not result["success"]:
-        raise SystemExit(124)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=remove.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--yes', # '-y',
+        action='store_true',
+        help='Remove links instantly without prompting to confirm.',
+    )
+    parser.add_argument(
+        '--delete', # '-r',
+        action='store_true',
+        help=(
+            "In addition to removing the link from the index, "
+            "also delete its archived content and metadata folder."
+        ),
+    )
+    parser.add_argument(
+        '--before', #'-b',
+        type=float,
+        help="List only URLs bookmarked before the given timestamp.",
+        default=None,
+    )
+    parser.add_argument(
+        '--after', #'-a',
+        type=float,
+        help="List only URLs bookmarked after the given timestamp.",
+        default=None,
+    )
+    parser.add_argument(
+        '--filter-type',
+        type=str,
+        choices=('exact', 'substring', 'domain', 'regex','tag'),
+        default='exact',
+        help='Type of pattern matching to use when filtering URLs',
+    )
+    parser.add_argument(
+        'filter_patterns',
+        nargs='*',
+        type=str,
+        help='URLs matching this filter pattern will be removed from the index.'
+    )
+    command = parser.parse_args(args or ())
+    
+    filter_str = None
+    if not command.filter_patterns:
+        filter_str = accept_stdin(stdin)

+    remove(
+        filter_str=filter_str,
+        filter_patterns=command.filter_patterns,
+        filter_type=command.filter_type,
+        before=command.before,
+        after=command.after,
+        yes=command.yes,
+        delete=command.delete,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
+    )
+    

-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,483 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox run [--daemon] [--crawl-id=...] [--snapshot-id=...] [--binary-id=...]
-
-Unified command for processing queued work on the shared abx-dl bus.
-
-Modes:
-    - With stdin JSONL: Process piped records, exit when complete
-    - Without stdin (TTY): Run the background runner in foreground until killed
-    - --crawl-id: Run the crawl runner for a specific crawl only
-    - --snapshot-id: Run a specific snapshot through its parent crawl
-    - --binary-id: Emit a BinaryRequestEvent for a specific Binary row
-
-Examples:
-    # Run the background runner in foreground
-    archivebox run
-
-    # Run as daemon (don't exit on idle)
-    archivebox run --daemon
-
-    # Process specific records (pipe any JSONL type, exits when done)
-    archivebox snapshot list --status=queued | archivebox run
-    archivebox archiveresult list --status=failed | archivebox run
-    archivebox crawl list --status=queued | archivebox run
-
-    # Mixed types work too
-    cat mixed_records.jsonl | archivebox run
-
-    # Run the crawl runner for a specific crawl
-    archivebox run --crawl-id=019b7e90-04d0-73ed-adec-aad9cfcd863e
-
-    # Run one snapshot from an existing crawl
-    archivebox run --snapshot-id=019b7e90-5a8e-712c-9877-2c70eebe80ad
-
-    # Run one queued binary install directly on the bus
-    archivebox run --binary-id=019b7e90-5a8e-712c-9877-2c70eebe80ad
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox run"
-
-import asyncio
-import os
-import signal
-import sys
-from collections import defaultdict
-
-import rich_click as click
-from rich import print as rprint
-
-
-RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
-
-
-def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
-    os._exit(128 + int(sig))
-
-
-def process_stdin_records() -> int:
-    """
-    Process JSONL records from stdin.
-
-    Create-or-update behavior:
-    - Records WITHOUT id: Create via Model.from_json(), then queue
-    - Records WITH id: Lookup existing, re-queue for processing
-
-    Outputs JSONL of all processed records (for chaining).
-
-    Handles any record type: Crawl, Snapshot, ArchiveResult.
-    Auto-cascades: Crawl → Snapshots → ArchiveResults.
-
-    Returns exit code (0 = success, 1 = error).
-    """
-    from django.utils import timezone
-
-    from archivebox.misc.jsonl import (
-        read_stdin,
-        write_record,
-        TYPE_CRAWL,
-        TYPE_SNAPSHOT,
-        TYPE_ARCHIVERESULT,
-        TYPE_BINARYREQUEST,
-        TYPE_BINARY,
-    )
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.core.models import Snapshot, ArchiveResult
-    from archivebox.crawls.models import Crawl
-    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-    from archivebox.machine.models import Binary
-    from archivebox.services.runner import run_binary, run_crawl
-
-    records = list(read_stdin())
-    is_tty = sys.stdout.isatty()
-
-    if not records:
-        return 0  # Nothing to process
-
-    created_by_id = get_or_create_system_user_pk()
-    queued_count = 0
-    output_records = []
-    full_crawl_ids: set[str] = set()
-    snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
-    plugin_names_by_crawl: dict[str, set[str]] = defaultdict(set)
-    run_all_plugins_for_crawl: set[str] = set()
-    binary_ids: list[str] = []
-
-    for record in records:
-        record_type = record.get("type", "")
-        record_id = record.get("id")
-
-        try:
-            if record_type == TYPE_CRAWL:
-                if record_id:
-                    # Existing crawl - re-queue
-                    try:
-                        crawl = Crawl.objects.get(id=record_id)
-                    except Crawl.DoesNotExist:
-                        crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
-                else:
-                    # New crawl - create it
-                    crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
-
-                if crawl:
-                    crawl.update_and_requeue(
-                        status=Crawl.StatusChoices.QUEUED,
-                        retry_at=timezone.now(),
-                    )
-                    full_crawl_ids.add(str(crawl.id))
-                    run_all_plugins_for_crawl.add(str(crawl.id))
-                    output_records.append(crawl.to_json())
-                    queued_count += 1
-
-            elif record_type == TYPE_SNAPSHOT or (record.get("url") and not record_type):
-                if record_id:
-                    # Existing snapshot - re-queue
-                    try:
-                        snapshot = Snapshot.objects.get(id=record_id)
-                    except Snapshot.DoesNotExist:
-                        snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
-                else:
-                    # New snapshot - create it
-                    snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
-
-                if snapshot:
-                    snapshot.queue_for_extraction()
-                    crawl_id = str(snapshot.crawl_id)
-                    snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
-                    run_all_plugins_for_crawl.add(crawl_id)
-                    output_records.append(snapshot.to_json())
-                    queued_count += 1
-
-            elif record_type == TYPE_ARCHIVERESULT:
-                if record_id:
-                    # Existing archiveresult - re-queue
-                    try:
-                        archiveresult = ArchiveResult.objects.get(id=record_id)
-                    except ArchiveResult.DoesNotExist:
-                        archiveresult = None
-                else:
-                    archiveresult = None
-
-                snapshot_id = record.get("snapshot_id")
-                plugin_name = record.get("plugin")
-                snapshot = None
-                if archiveresult:
-                    if archiveresult.status in [
-                        ArchiveResult.StatusChoices.FAILED,
-                        ArchiveResult.StatusChoices.SKIPPED,
-                        ArchiveResult.StatusChoices.NORESULTS,
-                        ArchiveResult.StatusChoices.BACKOFF,
-                    ]:
-                        archiveresult.reset_for_retry()
-                    snapshot = archiveresult.snapshot
-                    plugin_name = plugin_name or archiveresult.plugin
-                elif snapshot_id:
-                    try:
-                        snapshot = Snapshot.objects.get(id=snapshot_id)
-                    except Snapshot.DoesNotExist:
-                        snapshot = None
-
-                if snapshot:
-                    snapshot.queue_for_extraction()
-                    crawl_id = str(snapshot.crawl_id)
-                    snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
-                    if plugin_name:
-                        plugin_names_by_crawl[crawl_id].add(str(plugin_name))
-                    output_records.append(record if not archiveresult else archiveresult.to_json())
-                    queued_count += 1
-
-            elif record_type in {TYPE_BINARYREQUEST, TYPE_BINARY}:
-                if record_id:
-                    try:
-                        binary = Binary.objects.get(id=record_id)
-                    except Binary.DoesNotExist:
-                        binary = Binary.from_json(record)
-                else:
-                    binary = Binary.from_json(record)
-
-                if binary:
-                    binary.retry_at = timezone.now()
-                    if binary.status != Binary.StatusChoices.INSTALLED:
-                        binary.status = Binary.StatusChoices.QUEUED
-                    binary.save()
-                    binary_ids.append(str(binary.id))
-                    output_records.append(binary.to_json())
-                    queued_count += 1
-
-            else:
-                # Unknown type - pass through
-                output_records.append(record)
-
-        except Exception as e:
-            rprint(f"[yellow]Error processing record: {e}[/yellow]", file=sys.stderr)
-            continue
-
-    # Output all processed records (for chaining)
-    if not is_tty:
-        for rec in output_records:
-            write_record(rec)
-
-    if queued_count == 0:
-        rprint("[yellow]No records to process[/yellow]", file=sys.stderr)
-        return 0
-
-    rprint(f"[blue]Processing {queued_count} records...[/blue]", file=sys.stderr)
-
-    for binary_id in binary_ids:
-        run_binary(binary_id)
-
-    targeted_crawl_ids = full_crawl_ids | set(snapshot_ids_by_crawl)
-    if targeted_crawl_ids:
-        for crawl_id in sorted(targeted_crawl_ids):
-            try:
-                crawl = Crawl.objects.get(id=crawl_id)
-            except Crawl.DoesNotExist:
-                continue
-            if not crawl.claim_processing_lock(lock_seconds=10):
-                rprint(f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]", file=sys.stderr)
-                return 1
-            with foreground_shutdown_signals(), foreground_parent_watchdog():
-                run_crawl(
-                    crawl_id,
-                    snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
-                    selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
-                    selected_plugins_are_explicit=False,
-                )
-    return 0
-
-
-def run_runner(
-    daemon: bool = False,
-    crawl_id: str | None = None,
-    maintenance_only: bool = False,
-    maintenance_batch_size: int | None = None,
-) -> int:
-    """
-    Run the background runner loop.
-
-    Args:
-        daemon: Run forever (don't exit when idle)
-
-    Returns exit code (0 = success, 1 = error).
-    """
-    from archivebox.config import CONSTANTS
-    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-    from archivebox.machine.models import Machine, Process
-    from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
-    from archivebox.core.recovery_util import recover_orchestrator_state
-    from archivebox.services.runner import run_pending_crawls
-
-    Machine.current()
-    current = Process.current()
-    root_command = current.root
-    if daemon and root_command.process_type in (
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ADD,
-        Process.TypeChoices.UPDATE,
-    ):
-        # Server-owned daemon runners are persistent supervisor workers, but
-        # foreground add/update commands are allowed to borrow runner/sonic
-        # leadership without taking down Daphne. Waiting here keeps the worker
-        # on the normal runner path while preventing a server restart loop from
-        # immediately stealing the single-runner gate back from the newer CLI.
-        standby_until_foreground_runner_needed(root_command, data_dir=CONSTANTS.DATA_DIR)
-    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
-        current.mark_exited()
-        return 0
-
-    recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
-    if crawl_id:
-        from django.utils import timezone
-        from archivebox.crawls.models import Crawl
-
-        crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
-        now = timezone.now()
-        # Only re-lease when the row is unscheduled (retry_at IS NULL) or its
-        # existing lease has already expired. A future retry_at means another
-        # worker is already scheduled — don't clobber.
-        if crawl is not None and (crawl.retry_at is None or crawl.retry_at <= now):
-            # extra_filter pins the read-time retry_at so a concurrent worker
-            # that grabbed the lease between our SELECT and UPDATE wins.
-            crawl.safe_update(
-                {"retry_at": now},
-                refresh=False,
-                extra_filter={"retry_at": crawl.retry_at},
-            )
-    # Only a foreground `archivebox add` gets the interactive "abort current
-    # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
-    # orchestrators should shut down immediately and cleanly on the first signal.
-    interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
-    if daemon:
-        os.environ[RUNNER_DAEMON_ENV] = "1"
-    try:
-        with (
-            foreground_shutdown_signals(
-                on_signal=_exit_daemon_runner_on_signal if daemon else None,
-                raise_on_first_signal=not daemon,
-            ),
-            foreground_parent_watchdog(enabled=not daemon),
-        ):
-            run_pending_crawls(
-                daemon=daemon,
-                crawl_id=crawl_id,
-                maintenance_only=maintenance_only,
-                interactive_interrupts=interactive_interrupts,
-                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
-            )
-        return 0
-    except KeyboardInterrupt:
-        return 0
-    except asyncio.CancelledError as e:
-        if daemon:
-            rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-            return 1
-        return 0
-    except Exception as e:
-        rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-        return 1
-    finally:
-        current.refresh_from_db()
-        if current.status != Process.StatusChoices.EXITED:
-            current.mark_exited()
-
-
-@click.command()
-@click.option("--daemon", "-d", is_flag=True, help="Run forever (don't exit on idle)")
-@click.option("--crawl-id", help="Run the crawl runner for a specific crawl only")
-@click.option("--snapshot-id", help="Run one snapshot through its crawl")
-@click.option("--binary-id", help="Run one queued binary install directly on the bus")
-@click.option("--maintenance-only", is_flag=True, help="Only process sealed Snapshot maintenance and search-index backfills")
-@click.option(
-    "--maintenance-batch-size",
-    type=int,
-    default=None,
-    hidden=True,
-    help="Limit queued maintenance snapshots claimed per scheduler tick",
-)
-@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
-def main(
-    daemon: bool,
-    crawl_id: str,
-    snapshot_id: str,
-    binary_id: str,
-    maintenance_only: bool,
-    maintenance_batch_size: int | None,
-    no_stdin: bool,
-):
-    """
-    Process queued work.
-
-    Modes:
-    - No args + stdin piped: Process piped JSONL records
-    - No args + TTY: Run the crawl runner for all work
-    - --crawl-id: Run the crawl runner for that crawl only
-    - --snapshot-id: Run one snapshot through its crawl only
-    - --binary-id: Run one queued binary install directly on the bus
-    """
-    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-
-    if daemon and not snapshot_id and not binary_id and not crawl_id:
-        try:
-            os.environ[RUNNER_DAEMON_ENV] = "1"
-            with (
-                foreground_shutdown_signals(
-                    on_signal=_exit_daemon_runner_on_signal,
-                    raise_on_first_signal=False,
-                ),
-                foreground_parent_watchdog(enabled=False),
-            ):
-                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
-        except KeyboardInterrupt:
-            sys.exit(0)
-
-    with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
-        if snapshot_id:
-            sys.exit(run_snapshot_worker(snapshot_id))
-
-        if binary_id:
-            try:
-                from archivebox.services.runner import run_binary
-
-                run_binary(binary_id)
-                sys.exit(0)
-            except KeyboardInterrupt:
-                sys.exit(0)
-            except Exception as e:
-                rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-                import traceback
-
-                traceback.print_exc()
-                sys.exit(1)
-
-        if crawl_id:
-            sys.exit(
-                run_runner(
-                    daemon=False,
-                    crawl_id=crawl_id,
-                    maintenance_only=maintenance_only,
-                    maintenance_batch_size=maintenance_batch_size,
-                ),
-            )
-
-        if maintenance_only:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))
-
-        if not no_stdin and not sys.stdin.isatty():
-            sys.exit(process_stdin_records())
-        else:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
-
-
-def run_snapshot_worker(snapshot_id: str) -> int:
-    from archivebox.config import CONSTANTS
-    from archivebox.core.takeover_util import enter_single_runner_gate
-    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-    from archivebox.machine.models import Process
-    from archivebox.core.models import Snapshot
-    from archivebox.services.runner import run_due_snapshot
-    from django.utils import timezone
-
-    current = Process.current()
-    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
-        current.mark_exited()
-        return 0
-
-    snapshot = None
-    try:
-        with foreground_shutdown_signals(), foreground_parent_watchdog():
-            for _ in range(10):
-                snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-                if snapshot.retry_at is None:
-                    snapshot.update_and_requeue(retry_at=timezone.now())
-                elif snapshot.retry_at > timezone.now():
-                    break
-                if not run_due_snapshot(snapshot, lock_seconds=60):
-                    break
-        return 0
-    except KeyboardInterrupt:
-        try:
-            if snapshot is not None:
-                snapshot.refresh_from_db()
-            else:
-                snapshot = Snapshot.objects.filter(id=snapshot_id).first()
-            if snapshot is not None and snapshot.status != Snapshot.StatusChoices.SEALED:
-                snapshot.update_and_requeue(retry_at=timezone.now())
-        except Exception:
-            pass
-        return 0
-    except Exception as e:
-        rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-        import traceback
-
-        traceback.print_exc()
-        return 1
-    finally:
-        current.refresh_from_db()
-        if current.status != Process.StatusChoices.EXITED:
-            current.mark_exited()
-
-
-if __name__ == "__main__":
-    main()
@@ -1,186 +1,116 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox schedule'

-import rich_click as click
-from rich import print
+import sys
+import argparse
+from pathlib import Path
+from typing import Optional, List, IO

-from archivebox.misc.util import enforce_types, docstring
+from archivebox.misc.util import docstring
+from archivebox.config import DATA_DIR
+from ..logging_util import SmartFormatter, reject_stdin
+from ..main import schedule


-@enforce_types
-def schedule(
-    add: bool = False,
-    show: bool = False,
-    clear: bool = False,
-    foreground: bool = False,
-    run_all: bool = False,
-    quiet: bool = False,
-    every: str | None = None,
-    tag: str = "",
-    depth: int | str = 0,
-    import_path: str | None = None,
-    config: dict[str, object] | None = None,
-):
-    """Manage database-backed scheduled crawls processed by the crawl runner."""
-
-    from django.utils import timezone
-
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.crawls.models import Crawl, CrawlSchedule
-    from archivebox.crawls.schedule_util import validate_schedule
-    from archivebox.services.runner import run_pending_crawls
-
-    config_overrides = dict(config or {})
-    depth = int(depth)
-    result: dict[str, object] = {
-        "created_schedule_ids": [],
-        "disabled_count": 0,
-        "run_all_enqueued": 0,
-        "active_schedule_ids": [],
-    }
-
-    def _active_schedules():
-        return CrawlSchedule.objects.filter(is_enabled=True).select_related("template").order_by("created_at")
-
-    if clear:
-        disabled_count = CrawlSchedule.objects.filter(is_enabled=True).update(
-            is_enabled=False,
-            modified_at=timezone.now(),
-        )
-        result["disabled_count"] = disabled_count
-        print(f"[green]\\[√] Disabled {disabled_count} scheduled crawl(s).[/green]")
-
-    if every or add:
-        schedule_str = (every or "day").strip()
-        validate_schedule(schedule_str)
-
-        created_by_id = get_or_create_system_user_pk()
-        is_update_schedule = not import_path
-        template_urls = import_path or "archivebox://update"
-        template_label = (f"Scheduled import: {template_urls}" if import_path else "Scheduled ArchiveBox update")[:64]
-        template_notes = (
-            f"Created by archivebox schedule for {template_urls}"
-            if import_path
-            else "Created by archivebox schedule to queue recurring archivebox://update maintenance crawls."
-        )
-
-        template = Crawl.objects.create(
-            urls=template_urls,
-            max_depth=0 if is_update_schedule else depth,
-            tags_str="" if is_update_schedule else tag,
-            label=template_label,
-            notes=template_notes,
-            created_by_id=created_by_id,
-            status=Crawl.StatusChoices.SEALED,
-            retry_at=None,
-            config={
-                "DEPTH": 0 if is_update_schedule else depth,
-                "SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
-                # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the
-                # template defaults. Anything left unset falls through to the
-                # standard config stack at crawl-resolution time.
-                **config_overrides,
-            },
-        )
-        crawl_schedule = CrawlSchedule.objects.create(
-            template=template,
-            schedule=schedule_str,
-            is_enabled=True,
-            label=template_label,
-            notes=template_notes,
-            created_by_id=created_by_id,
-        )
-        result["created_schedule_ids"] = [str(crawl_schedule.id)]
-
-        schedule_type = "maintenance update" if is_update_schedule else "crawl"
-        print(f"[green]\\[√] Created scheduled {schedule_type}.[/green]")
-        print(f"    id={crawl_schedule.id}")
-        print(f"    every={crawl_schedule.schedule}")
-        print(f"    next_run={crawl_schedule.next_run_at.isoformat()}")
-        if import_path:
-            print(f"    source={import_path}")
-
-    schedules = list(_active_schedules())
-    result["active_schedule_ids"] = [str(schedule.id) for schedule in schedules]
-
-    if show:
-        if schedules:
-            print(f"[green]\\[*] Active scheduled crawls: {len(schedules)}[/green]")
-            for scheduled_crawl in schedules:
-                template = scheduled_crawl.template
-                print(
-                    f"  - id={scheduled_crawl.id} every={scheduled_crawl.schedule} "
-                    f"next_run={scheduled_crawl.next_run_at.isoformat()} "
-                    f"source={template.urls.splitlines()[0] if template.urls else ''}",
-                )
-        else:
-            print("[yellow]\\[*] No scheduled crawls are enabled.[/yellow]")
-
-    if run_all:
-        enqueued = 0
-        now = timezone.now()
-        for scheduled_crawl in schedules:
-            scheduled_crawl.enqueue(queued_at=now)
-            enqueued += 1
-        result["run_all_enqueued"] = enqueued
-        print(f"[green]\\[*] Enqueued {enqueued} scheduled crawl(s) immediately.[/green]")
-        if enqueued:
-            print(
-                "[yellow]\\[*] Start `archivebox server`, `archivebox run --daemon`, or `archivebox schedule --foreground` to process the queued crawls.[/yellow]",
-            )
-
-    if foreground:
-        print(
-            "[green]\\[*] Starting global crawl runner in foreground mode. It will materialize scheduled crawls and process queued work.[/green]",
-        )
-        run_pending_crawls(daemon=True)
-
-    if quiet:
-        return result
-
-    if not any((every, add, show, clear, foreground, run_all)):
-        if schedules:
-            print("[green]\\[*] Active scheduled crawls:[/green]")
-            for scheduled_crawl in schedules:
-                print(f"  - {scheduled_crawl.id} every={scheduled_crawl.schedule} next_run={scheduled_crawl.next_run_at.isoformat()}")
-        else:
-            print("[yellow]\\[*] No scheduled crawls are enabled.[/yellow]")
-
-    return result
-
-
-@click.command()
-@click.option("--quiet", "-q", is_flag=True, help="Return structured results without extra summary output")
-@click.option("--add", is_flag=True, help="Create a new scheduled crawl")
-@click.option("--every", type=str, help='Run on an alias like daily/weekly/monthly or a cron expression such as "0 */6 * * *"')
-@click.option("--tag", "-t", default="", help="Comma-separated tags to apply to scheduled crawl snapshots")
-@click.option(
-    "--depth",
-    type=click.Choice([str(i) for i in range(5)]),
-    default="0",
-    help="Recursively archive linked pages up to N hops away",
-)
-@click.option(
-    "--only-new/--no-only-new",
-    "only_new",
-    default=None,
-    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
-    "Pass --no-only-new to force re-archive on each scheduled run.",
-)
-@click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
-@click.option("--show", is_flag=True, help="Print all currently enabled schedules")
-@click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
-@click.option("--run-all", is_flag=True, help="Enqueue all enabled schedules immediately and process them once")
-@click.argument("import_path", required=False)
@docstring(schedule.__doc__)
-def main(**kwargs):
-    """Manage database-backed scheduled crawls processed by the crawl runner."""
-    only_new = kwargs.pop("only_new", None)
-    if only_new is not None:
-        kwargs["config"] = {"ONLY_NEW": bool(only_new)}
-    schedule(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=schedule.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--quiet', '-q',
+        action='store_true',
+        help=("Don't warn about storage space."),
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--add', # '-a',
+        action='store_true',
+        help='Add a new scheduled ArchiveBox update job to cron',
+    )
+    parser.add_argument(
+        '--every', # '-e',
+        type=str,
+        default=None,
+        help='Run ArchiveBox once every [timeperiod] (hour/day/month/year or cron format e.g. "0 0 * * *")',
+    )
+    parser.add_argument(
+        '--tag', '-t',
+        type=str,
+        default='',
+        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
+    )
+    parser.add_argument(
+        '--depth', # '-d',
+        type=int,
+        choices=[0, 1],
+        default=0,
+        help='Depth to archive to [0] or 1, see "add" command help for more info',
+    )
+    parser.add_argument(
+        '--overwrite',
+        action='store_true',
+        help='Re-archive any URLs that have been previously archived, overwriting existing Snapshots',
+    )
+    parser.add_argument(
+        '--update',
+        action='store_true',
+        help='Re-pull any URLs that have been previously added, as needed to fill missing ArchiveResults',
+    )
+    group.add_argument(
+        '--clear', # '-c'
+        action='store_true',
+        help=("Stop all ArchiveBox scheduled runs (remove cron jobs)"),
+    )
+    group.add_argument(
+        '--show', # '-s'
+        action='store_true',
+        help=("Print a list of currently active ArchiveBox cron jobs"),
+    )
+    group.add_argument(
+        '--foreground', '-f',
+        action='store_true',
+        help=("Launch ArchiveBox scheduler as a long-running foreground task "
+              "instead of using cron."),
+    )
+    group.add_argument(
+        '--run-all', # '-a',
+        action='store_true',
+        help=("Run all the scheduled jobs once immediately, independent of "
+              "their configured schedules, can be used together with --foreground"),
+    )
+    parser.add_argument(
+        'import_path',
+        nargs='?',
+        type=str,
+        default=None,
+        help=("Check this path and import any new links on every run "
+              "(can be either local file or remote URL)"),
+    )
+    command = parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)
+
+    schedule(
+        add=command.add,
+        show=command.show,
+        clear=command.clear,
+        foreground=command.foreground,
+        run_all=command.run_all,
+        quiet=command.quiet,
+        every=command.every,
+        tag=command.tag,
+        depth=command.depth,
+        overwrite=command.overwrite,
+        update=command.update,
+        import_path=command.import_path,
+        out_dir=Path(pwd) if pwd else DATA_DIR,
+    )


-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,10 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox search"
-
-from archivebox.cli.archivebox_list import main
-
-
-if __name__ == "__main__":
-    main()