fix docker release script

bump more js and py dependency lockfiles
bump dependencies and metadata for 0.7.3
2024-12-15 02:06:49 -08:00 · 2024-12-15 00:33:03 -08:00 · 2024-12-15 00:25:35 -08:00
496 changed files with 20325 additions and 95752 deletions
@@ -1,5 +0,0 @@
-{
-  "enabledPlugins": {
-    "pyright-lsp@claude-plugins-official": true
-  }
-}
@@ -1,51 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(**)",
-      "Glob(**)",
-      "Grep(**)",
-      "Bash(python -m archivebox:*)",
-      "Bash(ls:*)",
-      "Bash(xargs:*)",
-      "Bash(python -c:*)",
-      "Bash(printf:*)",
-      "Bash(pkill:*)",
-      "Bash(python3:*)",
-      "Bash(sqlite3:*)",
-      "WebFetch(domain:github.com)",
-      "Bash(uv add:*)",
-      "Bash(mkdir:*)",
-      "Bash(chmod:*)",
-      "Bash(python -m forum_dl:*)",
-      "Bash(archivebox manage migrate:*)",
-      "Bash(cat:*)",
-      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
-      "Bash(forum-dl:*)",
-      "Bash(pip uninstall:*)",
-      "Bash(python:*)",
-      "Bash(source .venv/bin/activate)",
-      "Bash(mv:*)",
-      "Bash(echo:*)",
-      "Bash(grep:*)",
-      "WebFetch(domain:python-statemachine.readthedocs.io)",
-      "Bash(./bin/run_plugin_tests.sh:*)",
-      "Bash(done)",
-      "Bash(coverage erase:*)",
-      "Bash(gh api:*)"
-    ]
-  },
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
-            "statusMessage": "Checking working directory..."
-          }
-        ]
-      }
-    ]
-  }
-}
@@ -9,22 +9,14 @@ __pycache__/
 .pdm-python
 .eggs/
 .git/
-.vscode/
 !.git/HEAD
 !.git/refs/heads/*

 venv/
 .venv/
 .venv-old/
-.docker_venv/
 .docker-venv/
 node_modules/
-chrome/
-chromeprofile/
-chrome_profile/
-
-pdm.dev.lock
-pdm.lock

 docs/
 build/
@@ -34,12 +26,6 @@ deb_dist/
 pip_dist/
 assets/
 docker/
-website/
-typings/

-tmp/
 data/
-data*/
 output/
-index.sqlite3
-index.sqlite3-wal
@@ -1,26 +0,0 @@
-# Read the Docs config for https://docs.archivebox.io
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
-
-version: 2
-
-submodules:
-  include: all
-  recursive: true
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.12"
-    #nodejs: "20"     # not needed unless we need the full archivebox to run while building docs for some reason
-
-sphinx:
-  configuration: docs/conf.py
-
-formats:
-  - pdf
-  - epub
-
-# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-python:
-  install:
-    - requirements: docs/requirements.txt
@@ -1,2 +1,3 @@
-github: ["ArchiveBox", "pirate"]
-custom: ["https://donate.archivebox.io", "https://swag.archivebox.io"]
+github: pirate
+patreon: theSquashSH
+custom: ["https://twitter.com/ArchiveBoxApp", "https://paypal.me/NicholasSweeting", "https://www.blockchain.com/eth/address/0x5D4c34D4a121Fe08d1dDB7969F07550f2dB9f471", "https://www.blockchain.com/btc/address/1HuxXriPE2Bbnag3jJrqa3bkNHrs297dYH"]
@@ -1,198 +0,0 @@
-name: 🐞 Bug report
-description: Report a bug or error you encountered in ArchiveBox
-title: "Bug: ..."
-assignees:
-  - pirate
-type: 'Bug'
-body:
-  - type: markdown
-    attributes:
-      value: |
-        *Please note:* it is normal to see errors occasionally for some extractors on some URLs (not every extractor will work on every type of page).  
-        Please report archiving errors if you are seeing them *consistently across many URLs* or if they are *preventing you from using ArchiveBox*.
-
-  - type: textarea
-    id: description
-    attributes:
-      label: Provide a screenshot and describe the bug
-      description: |
-        Attach a screenshot and describe what the issue is, what you expected to happen, and if relevant, the *URLs you were trying to archive*.  
-      placeholder: |
-        Got a bunch of 'singlefile was unable to archive this page' errors when trying to archive URLs from this site: https://example.com/xyz ...
-        I also tried to archive the same URLs using `singlefile` directly and some of them worked but not all of them. etc. ...
-    validations:
-      required: true
-
-  - type: textarea
-    id: steps_to_reproduce
-    attributes:
-      label: Steps to reproduce
-      description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
-      render: markdown
-      placeholder: |
-        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
-        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
-        3. Typed 'https://example.com/xyz' into the 'Add URL' input field
-        4. Clicked the 'Add+' button
-        5. Got a 500 error and saw the errors below in terminal
-    validations:
-      required: true
-
-  - type: textarea
-    id: logs
-    attributes:
-      label: Logs or errors
-      description: "Paste any terminal output, logs, or errors (check `data/logs/errors.log` as well)."
-      placeholder: |
-        ╭─────────────────────────────────────────────────────────────────────────────────────────────────────────╮
-        │ [2024-11-02 19:54:28] ArchiveBox v0.8.6rc0: archivebox add https://example.com#1234567                  │
-        ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯
-        
-        [+] [2024-11-02 19:54:29] Adding 1 links to index (crawl depth=0)...
-            > Saved verbatim input to sources/1730577269-import.txt
-            > Parsed 1 URLs from input (Generic TXT)
-            ...
-      render: shell
-    validations:
-      required: false
-
-  - type: textarea
-    id: version
-    attributes:
-      label: ArchiveBox Version
-      description: |
-        **REQUIRED:** Run the `archivebox version` command inside your collection dir and paste the *full output* here (*not just the version number*).  
-        For Docker Compose run: `docker compose run archivebox version`  
-        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
-      render: shell
-      placeholder: |
-        0.8.6
-        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
-        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
-        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
-        
-        Binary Dependencies:
-        √  python     3.14.0     venv_pip   ~/.venv/bin/python
-        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
-        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
-        ...
-    validations:
-      required: true
-
-  - type: dropdown
-    id: install_method
-    validations:
-      required: true
-    attributes:
-      label: How did you install the version of ArchiveBox you are using?
-      multiple: false
-      options:
-        - pip
-        - apt
-        - brew
-        - nix
-        - Docker (or Podman/LXC/K8s/TrueNAS/Proxmox/etc)
-        - Other
-
-  - type: dropdown
-    id: operating_system
-    validations:
-      required: true
-    attributes:
-      label: What operating system are you running on?
-      description: |
-        Please note we are *unable to provide support for Windows users* unless you are using [Docker on Windows](https://github.com/ArchiveBox/archivebox#:~:text=windows%20without%20docker).
-      multiple: false
-      options:
-        - Linux (Ubuntu/Debian/Arch/Alpine/etc.)
-        - macOS (including Docker on macOS)
-        - BSD (FreeBSD/OpenBSD/NetBSD/etc.)
-        - Windows (including WSL, WSL2, Docker Desktop on Windows)
-        - Other
-
-  - type: checkboxes
-    id: filesystem
-    attributes:
-      label: What type of drive are you using to store your ArchiveBox data?
-      description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` OR `data/archive`?
-      options:
-        - label: "some of `data/` is on a local SSD or NVMe drive"
-          required: false
-        - label: "some of `data/` is on a spinning hard drive or external USB drive"
-          required: false
-        - label: "some of `data/` is on a network mount (e.g. NFS/SMB/Ceph/GlusterFS/etc.)"
-          required: false
-        - label: "some of `data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/Google Drive/Dropbox/etc.)"
-          required: false
-  
-
-  - type: textarea
-    id: docker_compose_yml
-    attributes:
-      label: Docker Compose Configuration
-      description: "If using Docker Compose, please share your full `docker-compose.yml` file. If using plain Docker, paste the `docker run ...` command you use."
-      placeholder: |
-        services:
-            archivebox:
-                image: archivebox/archivebox:latest
-                ports:
-                    - 8000:8000
-                volumes:
-                    - ./data:/data
-                environment:
-                    - ADMIN_USERNAME=admin
-                    - ADMIN_PASSWORD=****<redact any passwords>****
-                    - ALLOWED_HOSTS=*
-                    - CSRF_TRUSTED_ORIGINS=https://archivebox.example.com
-                    - PUBLIC_INDEX=True
-                    - PUBLIC_SNAPSHOTS=True
-                    - PUBLIC_ADD_VIEW=False
-                    ...
-
-            archivebox_scheduler:
-                image: archivebox/archivebox:latest
-                command: schedule --foreground --update --every=day
-                environment:
-                    ...
-      
-          ...
-      render: shell
-    validations:
-      required: false
-
-  - type: textarea
-    id: configuration
-    attributes:
-      label: ArchiveBox Configuration
-      description: "Please share your full `data/ArchiveBox.conf` file here."
-      render: shell
-      placeholder: |
-        [SERVER_CONFIG]
-        SECRET_KEY = "*********<redact any secrets/passwords>************"
-
-        WGET_RESTRICT_FILE_NAMES=windows
-        USE_SYSTEM_WGET=true
-        CHECK_SSL_VALIDITY=false
-        ...
-    validations:
-      required: false
-
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying anything suggested here, **we are not responsible for data loss**.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
-          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
-          - 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
-          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
-        
@@ -1,128 +0,0 @@
-name: 💡 Feature or enhancement request
-description: Suggest an idea or improvement for this project
-title: "Feature Request: ..."
-assignees:
-  - pirate
-type: 'Enhancement'
-labels: 'status: idea phase'
-body:
-  - type: dropdown
-    id: suggestion_type
-    validations:
-      required: true
-    attributes:
-      label: "What type of suggestion are you making?"
-      multiple: false
-      options:
-        - New extractor / type of content to save
-        - Proposing a new feature
-        - Modification of existing behavior
-        - Web UI or UX design improvement
-
-  - type: textarea
-    id: current_problem
-    attributes:
-      label: "What is the problem that your feature request solves?"
-      description: |
-        Describe the problem or need that your feature request solves, feel free to include any screenshots or examples.
-      placeholder: |
-        e.g. I need to be able to archive spanish and french subtitle files from a particular movie site https://example.com/somevideos that's going down soon.
-    validations:
-      required: true
-
-  - type: textarea
-    id: proposed_solution
-    attributes:
-      label: "What is your proposed solution?"
-      description: |
-        Describe the ideal specific solution you'd want, *and whether it fits into any broader scope of changes*.
-      placeholder: |
-        e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
-        The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
-    validations:
-      required: true
-
-  - type: textarea
-    id: workarounds_tried
-    attributes:
-      label: "What hacks or alternative solutions have you tried to solve the problem?"
-      description: |
-        A description of any alternative approaches, workarounds, or other solutions you've considered to fix the problem.
-      placeholder: |
-        e.g. I wait for archivebox to finish archiving the page, then I manually run `yt-dlp --subs <URL>` inside 
-        the `data/archive/<timestamp>/` directory to download the subtitle files and add them to the snapshot folder.
-    validations:
-      required: true
-
-  - type: textarea
-    id: version
-    attributes:
-      label: Share the entire output of the `archivebox version` command for the current version you are using.
-      description: |
-        DO NOT JUST ENTER "the latest version" OR YOUR ISSUE WILL BE CLOSED.  
-        We need to know what version of ArchiveBox and what feature flags you're currently running with in order to contextualize your feature request.  
-        Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in your specific environment.
-          
-        Run the `archivebox version` command inside your current collection dir and paste the *full output* here (*not just the version number*).  
-        For Docker Compose run: `docker compose run archivebox version`  
-        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
-      render: shell
-      placeholder: |
-        0.8.6
-        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
-        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
-        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
-        
-        Binary Dependencies:
-        √  python     3.14.0     venv_pip   ~/.venv/bin/python
-        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
-        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
-        ...
-    validations:
-      required: true
-
-  - type: checkboxes
-    id: priority
-    attributes:
-      label: "How badly do you want this new feature?"
-      options:
-        - label: "It's an urgent deal-breaker, I can't live without it"
-          required: false
-        - label: "It's important to add it in the near-mid term future"
-          required: false
-        - label: "It would be nice to have eventually"
-          required: false
-        - label: "I'm willing to [start a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
-          required: false
-        - label: "I have [donated money](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations) to go towards fixing this issue"
-          required: false
-  
-  - type: checkboxes
-    id: satisfaction_survey
-    attributes:
-      label: Mini Survey
-      description: How do you like ArchiveBox so far?
-      options:
-        - label: "I like ArchiveBox so far / would recommend it to a friend"
-          required: false
-        - label: "I've had a lot of difficulty getting ArchiveBox set up"
-          required: false
-        - label: "I would pay $10/mo for a hosted version of ArchiveBox if it had this feature"
-          required: false
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying any workarounds or BETAs suggested here, **we are not responsible for data loss**.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 📊 [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
-          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
-          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
@@ -1,52 +0,0 @@
-name: 📑 Documentation improvement
-description: Submit an idea or correction for the Wiki documentation
-title: "Documentation: ..."
-labels: 'touches: docs'
-type: 'Enhancement'
-assignees:
-  - pirate
-body:
-  - type: markdown
-    attributes:
-      value: |
-        If you prefer, you can submit a [Pull Request](https://github.com/ArchiveBox/docs) on https://github.com/ArchiveBox/docs to edit the docs directly instead.
-
-  - type: input
-    id: page_url
-    validations:
-      required: true
-    attributes:
-      label: "What is the URL of the page you'd like to see improved?"
-      placeholder: e.g. https://github.com/ArchiveBox/docs/wiki/Install
-
-  - type: input
-    id: section_title
-    validations:
-      required: true
-    attributes:
-      label: "What is the title of the relevant section?"
-      placeholder: e.g. Option B. Automatic Setup Script
-
-  - type: textarea
-    id: suggested_edit
-    attributes:
-      label: "What is the suggested edit?"
-      placeholder: |
-        e.g. Please document how to run the automatic setup script for ArchiveBox on TempleOS.  
-        Attach images, screenshots, code snippets, etc. anything you think would help.
-    validations:
-      required: true
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to address issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
-          - 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
-          - ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)
@@ -0,0 +1,46 @@
+---
+name: 🐞 Bug report
+about: Create a report to help us improve
+title: 'Bug: ...'
+labels: 'bug'
+assignees: ''
+
+---
+
+<!--
+Please fill out the following information, 
+feel free to delete sections if they're not applicable 
+or if long issue templates annoy you.
+(the only required section is the version information)
+-->
+
+#### Describe the bug
+<!--
+A description of what the bug is, 
+what you expected to happen, 
+and any relevant context about issue.
+-->
+
+#### Steps to reproduce
+<!--
+For example:
+1. Ran ArchiveBox with the following config '...'
+2. Saw this output during archiving '....'
+3. UI didn't show the thing I was expecting '....'
+-->
+
+#### Screenshots or log output
+
+<!--
+If applicable, post any relevant screenshots or copy/pasted terminal output from ArchiveBox.
+If you're reporting a parsing / importing error, **you must paste a copy of your redacted import file here**.
+-->
+
+#### ArchiveBox version
+
+<!-- Run the `archivebox version` command locally then copy paste the result here: -->
+```logs
+replace this line with the *full*, unshortened output of running `archivebox version`
+```
+<!-- Tickets without full version info will closed until it is provided,
+we need the full output here to help you solve your issue -->
@@ -1,11 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: ❓ Ask a question or start a discussion
-    url: https://github.com/ArchiveBox/ArchiveBox/discussions
-    about: "Ask a question, get support, or start a design discussion (to report a problem please use '🐞 Bug report' instead)"
-  - name: 💬 Chat with the dev team & community on Zulip
-    url: https://zulip.archivebox.io
-    about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
-  - name: 💁‍♂️ Hire us for professional support with fast response times
-    url: https://docs.monadical.com/s/archivebox-consulting-services
-    about: "We provide hosting, development, and support, including on-prem/cloud w/ SSO & storage, CAPTCHA-solving, proxies, etc."
@@ -0,0 +1,16 @@
+---
+name: 📑 Documentation change
+about: Submit a suggestion for the Wiki documentation
+title: 'Documentation: Improvement request ...'
+labels: ''
+assignees: ''
+
+---
+
+## Wiki Page URL
+<!-- e.g. https://github.com/pirate/ArchiveBox/wiki/Configuration#use_color -->
+
+
+## Suggested Edit
+<!-- e.g. Please add more example usages, or please fix `xyz` typo to be `abc`. -->
+
@@ -0,0 +1,50 @@
+---
+name: 💡 Feature request
+about: Suggest an idea for this project
+title: 'Feature Request: ...'
+labels: 'changes: behavior,status: idea phase'
+assignees: ''
+
+---
+
+<!--
+Please fill out the following information, 
+feel free to delete sections if they're not applicable 
+or if long issue templates annoy you :)
+-->
+
+## Type 
+
+ - [ ] General question or discussion
+ - [ ] Propose a brand new feature
+ - [ ] Request modification of existing behavior or design
+
+## What is the problem that your feature request solves
+<!--
+e.g. I need to be able to archive spanish and french subtitle files  
+from a particular <example.com> movie site that's going down soon.
+-->
+
+## Describe the ideal specific solution you'd want, and whether it fits into any broader scope of changes
+<!--
+e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
+The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
+-->
+
+## What hacks or alternative solutions have you tried to solve the problem?
+<!--
+A clear and concise description of any alternative solutions, 
+workarounds, or other software you've considered using to fix the problem.
+-->
+
+## How badly do you want this new feature?
+
+ - [ ] It's an urgent deal-breaker, I can't live without it
+ - [ ] It's important to add it in the near-mid term future
+ - [ ] It would be nice to have eventually
+
+---
+
+ - [ ] I'm willing to contribute [dev time](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) / [money](https://github.com/sponsors/pirate) to fix this issue
+ - [ ] I like ArchiveBox so far / would recommend it to a friend
+ - [ ] I've had a lot of difficulty getting ArchiveBox set up
@@ -0,0 +1,9 @@
+---
+name: 💬 Question, discussion, or support request
+about: Start a discussion or ask a question about ArchiveBox
+title: 'Question: ...'
+labels: ''
+assignees: ''
+
+---
+
@@ -1,25 +0,0 @@
-# To get started with Dependabot version updates, you'll need to specify which
-# package ecosystems to update and where the package manifests are located.
-# Please see the documentation for all configuration options:
-# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
-
-version: 2
-updates:
-  - package-ecosystem: "pip"
-    directory: "/"
-    target-branch: "dev"
-    schedule:
-      interval: "monthly"
-    groups:
-      pip:
-        patterns:
-          - "*"
-  - package-ecosystem: "npm"
-    directory: "/"
-    target-branch: "dev"
-    schedule:
-      interval: "monthly"
-    groups:
-      npm:
-        patterns:
-          - "*"
@@ -1,49 +0,0 @@
-name: Claude Code
-
-on:
-  issue_comment:
-    types: [created]
-  pull_request_review_comment:
-    types: [created]
-  issues:
-    types: [opened, assigned]
-  pull_request_review:
-    types: [submitted]
-
-jobs:
-  claude:
-    if: |
-      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
-      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      pull-requests: write
-      issues: write
-      id-token: write
-      actions: read # Required for Claude to read CI results on PRs
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Run Claude Code
-        id: claude
-        uses: anthropics/claude-code-action@v1
-        with:
-          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
-
-          # This is an optional setting that allows Claude to read CI results on PRs
-          additional_permissions: |
-            actions: read
-
-          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
-          # prompt: 'Update the pull request description to include a summary of changes.'
-
-          # Optional: Add claude_args to customize behavior and configuration
-          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
-          # or https://code.claude.com/docs/en/cli-reference for available options
-          claude_args: '--allowed-tools Bash(gh pr:*)'
@@ -0,0 +1,32 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ dev ]
+  pull_request:
+    branches: [ dev ]
+  schedule:
+    - cron: '43 1 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        queries: security-extended
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1
@@ -1,92 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ "dev" ]
-  pull_request:
-    branches: [ "dev" ]
-  schedule:
-    - cron: '33 17 * * 6'
-
-jobs:
-  analyze:
-    name: Analyze (${{ matrix.language }})
-    # Runner size impacts CodeQL analysis time. To learn more, please see:
-    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
-    #   - https://gh.io/supported-runners-and-hardware-resources
-    #   - https://gh.io/using-larger-runners (GitHub.com only)
-    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
-    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
-    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
-    permissions:
-      # required for all workflows
-      security-events: write
-
-      # required to fetch internal or private CodeQL packs
-      packages: read
-
-      # only required for workflows in private repositories
-      actions: read
-      contents: read
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-        - language: python
-          build-mode: none
-        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
-        # Use `c-cpp` to analyze code written in C, C++ or both
-        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
-        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
-        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
-        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
-        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
-        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
-      with:
-        languages: ${{ matrix.language }}
-        build-mode: ${{ matrix.build-mode }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-    # If the analyze step fails for one of the languages you are analyzing with
-    # "We were unable to automatically build your code", modify the matrix above
-    # to set the build mode to "manual" for that language. Then modify this step
-    # to build your code.
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-    - if: matrix.build-mode == 'manual'
-      run: |
-        echo 'If you are using a "manual" build mode for one or more of the' \
-          'languages you are analyzing, replace this with the commands to build' \
-          'your code, for example:'
-        echo '  make bootstrap'
-        echo '  make release'
-        exit 1
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
-      with:
-        category: "/language:${{matrix.language}}"
@@ -2,209 +2,63 @@ name: Build Debian package

 on:
  workflow_dispatch:
-  workflow_call:
  push:
-    branches: ['**']
-    paths:
-      - 'pkg/debian/**'
-      - 'bin/build_deb.sh'
-      - 'bin/release_deb.sh'
-      - '.github/workflows/debian.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs

-permissions:
-  contents: write
+env:
+  DEB_BUILD_OPTIONS: nocheck

 jobs:
  build:
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        arch: [amd64, arm64]
+    runs-on: ubuntu-22.04

    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
        with:
+          submodules: true
          fetch-depth: 1

-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Install nfpm
-        env:
-          NFPM_VERSION: "2.45.1"
+      - name: Install packaging dependencies
        run: |
-          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
-            | sudo tar -xz -C /usr/local/bin nfpm
-          nfpm --version
+          sudo apt-get update -qq
+          sudo apt-get install -y \
+            python3 python3-dev python3-pip python3-venv python3-all \
+            dh-python debhelper devscripts dput software-properties-common \
+            python3-distutils python3-setuptools python3-wheel python3-stdeb

-      - name: Build .deb package
-        run: |
-          export VERSION="${{ steps.version.outputs.version }}"
-          export ARCH="${{ matrix.arch }}"
-          ./bin/build_deb.sh
+      # - name: Build Debian/Apt sdist_dsc
+      #   run: |
+      #     ./bin/build_pip.sh

-      - name: Verify .deb package contents
-        run: |
-          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
-          echo "=== Package info ==="
-          dpkg-deb --info "$DEB_FILE"
-          echo ""
-          echo "=== Package contents ==="
-          dpkg-deb --contents "$DEB_FILE"
-          echo ""
-          echo "=== Control fields ==="
-          dpkg-deb --field "$DEB_FILE"
+      # - name: Check ArchiveBox version
+      #   run: |
+      #     # must create dir needed for snaps to run as non-root on github actions
+      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
+      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
+      #     archivebox --version
+      #     archivebox init --setup

-      - name: Upload .deb artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
-          path: dist/*.deb
+      # - name: Add some links to test
+      #   run: |
+      #     cd "${{ github.workspace }}/data"
+      #     archivebox add 'https://example.com'
+      #     archivebox status

-  test:
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: ubuntu-24.04
-          - arch: arm64
-            runner: ubuntu-24.04-arm
-    runs-on: ${{ matrix.runner }}
+      # - name: Commit built package
+      #   run: |
+      #     cd deb_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Debian package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/debian-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: deb_dist

-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: true
-
-      - name: Install build dependencies
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local wheel
-        run: |
-          uv sync --frozen --all-extras --no-install-project --no-install-workspace
-          uv build --wheel --out-dir /tmp/wheels/
-
-      - name: Download .deb artifact
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*-${{ matrix.arch }}.deb
-          merge-multiple: true
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
-
-      - name: Pre-seed virtualenv with local wheel before dpkg install
-        run: |
-          # CI-only: pre-seed the venv with a local wheel so we test the
-          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
-          # here (matching the system dep installed above) to ensure the venv is
-          # created with the correct Python version. On real installs, install.sh
-          # handles this by preferring python3.13 and failing if < 3.13.
-          # When postinstall.sh runs during dpkg -i, it finds the venv already
-          # populated and just upgrades deps.
-          sudo mkdir -p /opt/archivebox
-          sudo python3.13 -m venv /opt/archivebox/venv
-          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
-          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
-          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
-
-      - name: Install .deb package
-        run: |
-          # dpkg install will run postinstall which creates the user,
-          # sets up systemd, and tries pip install (which finds it already installed)
-          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
-
-      - name: Verify archivebox is installed
-        run: |
-          which archivebox
-          archivebox version
-
-      - name: Verify wrapper script works
-        run: |
-          /usr/bin/archivebox version
-          /usr/bin/archivebox --help | head -5
-
-      - name: Test archivebox init as archivebox user
-        run: |
-          # The postinstall should have created the user
-          id archivebox
-          sudo mkdir -p /tmp/archivebox-test
-          sudo chown archivebox:archivebox /tmp/archivebox-test
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
-
-      - name: Test archivebox status
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
-
-      - name: Test archivebox add
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
-
-      - name: Verify systemd service file exists
-        run: |
-          test -f /usr/lib/systemd/system/archivebox.service
-          cat /usr/lib/systemd/system/archivebox.service
-
-  # Upload .deb to GitHub Release only after tests pass
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: [build, test]
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Determine release tag
-        id: tag
-        run: |
-          TAG="${{ github.event.release.tag_name }}"
-          if [ -z "$TAG" ]; then
-            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
-            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
-          fi
-          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-
-      - name: Download all .deb artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*.deb
-          merge-multiple: true
-
-      - name: Upload .deb to GitHub Release
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          TAG="${{ steps.tag.outputs.tag }}"
-          # Verify the release exists before uploading
-          if ! gh release view "$TAG" >/dev/null 2>&1; then
-            echo "[!] No GitHub Release found for tag $TAG."
-            if [ -n "${{ github.event.release.tag_name }}" ]; then
-              echo "[X] This was triggered by a release event — the release should exist. Failing."
-              exit 1
-            fi
-            echo "[i] Skipping upload (workflow_dispatch without a release)."
-            echo "    Create a release first or trigger via the release event."
-            exit 0
-          fi
-          gh release upload "$TAG" *.deb --clobber
+      # - name: Push build to Launchpad PPA
+      #   run: |
+      #     debsign -k "$PGP_KEY_ID" "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
+      #     dput archivebox "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
@@ -2,7 +2,6 @@ name: Build Docker image

 on:
  workflow_dispatch:
-  workflow_call:
  push:
    branches:
      - '**'
@@ -12,88 +11,61 @@ on:

 env:
  DOCKER_IMAGE: archivebox-ci
-
+      
 jobs:
  buildx:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
-        # with:
-        #   submodules: true
-        #   fetch-depth: 1
+        with:
+          submodules: true
+          fetch-depth: 1

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
-
+      
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: latest
          install: true
-          platforms: linux/amd64,linux/arm64
-
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+      
      - name: Builder instance name
        run: echo ${{ steps.buildx.outputs.name }}
-
+      
      - name: Available platforms
        run: echo ${{ steps.buildx.outputs.platforms }}
-
+      
      - name: Cache Docker layers
-        uses: actions/cache@v4
+        uses: actions/cache@v3
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-

-      - name: Login to Docker Hub
+      - name: Docker Login
        uses: docker/login-action@v3
        if: github.event_name != 'pull_request'
        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Collect Full Release Docker tags
-        # https://github.com/docker/metadata-action
+           username: ${{ secrets.DOCKER_USERNAME }}
+           password: ${{ secrets.DOCKER_PASSWORD }}
+      
+      - name: Collect Docker tags
        id: docker_meta
        uses: docker/metadata-action@v5
-        if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
        with:
-          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
+          images: archivebox/archivebox,nikisweeting/archivebox
          tags: |
-              # :stable
              type=ref,event=branch
-              # :0.7.3
              type=semver,pattern={{version}}
-              # :0.7
              type=semver,pattern={{major}}.{{minor}}
-              # :sha-463ea54
              type=sha
-              # :latest
-              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
-
-      - name: Collect Non-Release Docker tags
-        # https://github.com/docker/metadata-action
-        id: docker_meta_non_release
-        uses: docker/metadata-action@v5
-        if: github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call'
-        with:
-          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
-          tags: |
-              # :stable
-              type=ref,event=branch
-              # :sha-463ea54
-              type=sha
-
+              type=raw,value=latest,enable={{is_default_branch}}
+      
      - name: Build and push
        id: docker_build
        uses: docker/build-push-action@v5
@@ -102,22 +74,14 @@ jobs:
          file: ./Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.tags || steps.docker_meta_non_release.outputs.tags }}
-          labels: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.labels || steps.docker_meta_non_release.outputs.labels }}
+          tags: ${{ steps.docker_meta.outputs.tags }}
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache-new
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm64,linux/arm/v7

      - name: Image digest
        run: echo ${{ steps.docker_build.outputs.digest }}
-
-      - name: Update README
-        uses: peter-evans/dockerhub-description@v4
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: archivebox/archivebox
-
+       
      # This ugly bit is necessary if you don't want your cache to grow forever
      # until it hits GitHub's limit of 5GB.
      # Temp fix
@@ -1,59 +0,0 @@
-name: Duplicate Issue Detection
-
-on:
-  issues:
-    types: [opened]
-
-jobs:
-  check-duplicates:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Install opencode
-        run: curl -fsSL https://opencode.ai/install | bash
-
-      - name: Check for duplicate issues
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OPENCODE_PERMISSION: |
-            {
-              "bash": {
-                "gh issue*": "allow",
-                "gh pr*": "allow",
-                "*": "deny"
-              },
-              "webfetch": "allow"
-            }
-        run: |
-          opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created:
-
-          Issue number:
-          ${{ github.event.issue.number }}
-
-          Lookup this issue and search through existing issues and PRs (excluding #${{ github.event.issue.number }}) in this repository to find any potential duplicates of this new issue.
-          Consider:
-          1. Similar titles or descriptions
-          2. Same error messages or symptoms
-          3. Related functionality or components
-          4. Similar feature requests
-
-          If you find any potential duplicates, please comment on the new issue with:
-          - A brief explanation of why it might be a duplicate
-          - Links to the potentially duplicate issues or PRs
-          - A suggestion to check those issues first
-
-          Use this format for the comment:
-          'This issue might be a duplicate of existing issues. Please check:
-          - #[issue_number]: [brief description of similarity]
-
-          Feel free to ignore if none of these address your specific case.'
-
-          If no clear duplicates are found, do not comment."
@@ -1,64 +0,0 @@
-# Simple workflow for deploying static content to GitHub Pages
-name: Deploy static content to Pages
-
-on:
-  # Runs on pushes targeting the default branch
-  push:
-    branches: ["dev"]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
-# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
-concurrency:
-  group: "pages"
-  cancel-in-progress: false
-
-jobs:
-  # Single deploy job since we're just deploying
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - name: Copy README.md into place
-        run: |
-          rm -f ./website/README.md
-          cp ./README.md ./website/README.md
-      - name: Setup Pages
-        uses: actions/configure-pages@v5
-      - name: Build with Jekyll
-        uses: actions/jekyll-build-pages@v1
-        with:
-          source: ./website
-          destination: ./_site
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
-
-      # - name: Checkout
-      #   uses: actions/checkout@v4
-      # - name: Setup Pages
-      #   uses: actions/configure-pages@v5
-      # - name: Upload artifact
-      #   uses: actions/upload-pages-artifact@v3
-      #   with:
-      #     # Upload entire repository
-      #     path: './website'
-
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4
@@ -1,229 +1,51 @@
-name: Build Homebrew formula
+name: Build Homebrew package

 on:
  workflow_dispatch:
-  workflow_call:
  push:
-    branches: ['**']
-    paths:
-      - 'brew_dist/**'
-      - 'bin/build_brew.sh'
-      - 'bin/release_brew.sh'
-      - '.github/workflows/homebrew.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs

-permissions:
-  contents: read

 jobs:
-  build-and-test:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-latest, ubuntu-24.04]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Validate formula template syntax
-        run: ruby -c brew_dist/archivebox.rb
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: true
-
-      - name: Install build dependencies (Linux)
-        if: runner.os == 'Linux'
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local sdist
-        run: |
-          uv sync --frozen --all-extras --no-install-project --no-install-workspace
-          uv build --sdist --out-dir /tmp/sdist/
-
-      - name: Generate formula from local sdist
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
-          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
-
-          # Install archivebox + poet into a temp venv to generate resource stanzas
-          python3 -m venv /tmp/poet-venv
-          source /tmp/poet-venv/bin/activate
-          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
-          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
-          RESOURCES="$(poet archivebox)"
-          deactivate
-
-          # For CI: use file:// URL pointing to local sdist
-          # For release: this gets overridden with the PyPI URL
-          SDIST_URL="file://${SDIST_PATH}"
-
-          cat > /tmp/archivebox.rb << RUBY
-class Archivebox < Formula
-  include Language::Python::Virtualenv
-
-  desc "Self-hosted internet archiving solution"
-  homepage "https://github.com/ArchiveBox/ArchiveBox"
-  url "${SDIST_URL}"
-  sha256 "${SDIST_SHA256}"
-  license "MIT"
-
-  depends_on "python@3.13"
-
-${RESOURCES}
-
-  def install
-    virtualenv_install_with_resources
-  end
-
-  def post_install
-    data_dir = var/"archivebox"
-    data_dir.mkpath
-    ENV["DATA_DIR"] = data_dir.to_s
-    system bin/"archivebox", "init"
-  end
-
-  def caveats
-    <<~EOS
-      ArchiveBox data is stored in:
-        #{var}/archivebox
-
-      To start archiving, run:
-        cd #{var}/archivebox && archivebox add 'https://example.com'
-
-      To start the web UI:
-        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
-    EOS
-  end
-
-  test do
-    assert_match version.to_s, shell_output("#{bin}/archivebox version")
-  end
-end
-RUBY
-
-          echo "[√] Generated formula:"
-          ruby -c /tmp/archivebox.rb
-          cat /tmp/archivebox.rb
-
-      - name: Install Homebrew (Linux only)
-        if: runner.os == 'Linux'
-        run: |
-          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
-          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
-          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
-          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
-          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
-          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
-
-      - name: Install brew dependencies
-        run: brew install python@3.13
-
-      - name: Install archivebox via brew from local formula
-        run: |
-          brew install --build-from-source --verbose /tmp/archivebox.rb
-
-      - name: Verify archivebox version
-        run: archivebox version
-
-      - name: Test archivebox init
-        run: |
-          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
-          archivebox init
-
-      - name: Test archivebox status
-        run: |
-          cd /tmp/archivebox-test
-          archivebox status
-
-  # On release only: generate the real formula with PyPI URL and push to tap
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: build-and-test
+  build:
    runs-on: macos-latest

    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
        with:
+          submodules: true
          fetch-depth: 1

-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
+      # TODO: modify archivebox.rb to update src url, hashes, and dependencies

-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Wait for PyPI package availability
+      - name: Build Homebrew Bottle
        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
-          for i in $(seq 1 30); do
-            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
-              echo "[√] archivebox==${VERSION} is available on PyPI"
-              break
-            fi
-            if [ "$i" -eq 30 ]; then
-              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
-              break
-            fi
-            echo "    Attempt $i/30 - not yet available, waiting 30s..."
-            sleep 30
-          done
-
-      - name: Generate release formula via build_brew.sh
-        run: ./bin/build_brew.sh
-
-      - name: Test formula install
-        run: |
-          brew install --build-from-source brew_dist/archivebox.rb
+          pip3 install --upgrade pip setuptools wheel
+          cd brew_dist/
+          brew install --build-bottle ./archivebox.rb
+          # brew bottle archivebox
          archivebox version

-      - name: Upload formula artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox.rb
-          path: brew_dist/archivebox.rb
-
-      - name: Push to homebrew-archivebox tap
-        env:
-          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
+      - name: Add some links to test
        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap
+          mkdir data && cd data
+          archivebox init --setup
+          archivebox add 'https://example.com'
+          archivebox version
+          archivebox status

-          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
-            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb
+      # - name: Commit built package
+      #   run: |
+      #     cd brew_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Homebrew package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/homebrew-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: brew_dist

-          cd /tmp/tap
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add -A
-          if git diff --cached --quiet; then
-            echo "[i] No changes to formula, skipping push."
-          else
-            git commit -m "Update archivebox to v${VERSION}"
-            git push origin HEAD
-            echo "[√] Formula pushed to homebrew-archivebox tap"
-          fi
+      # TODO: push bottle homebrew core PR with latest changes
@@ -36,14 +36,14 @@ jobs:
          rm ./website/README.md
          cp ./README.md ./website/README.md
      - name: Setup Pages
-        uses: actions/configure-pages@v5
+        uses: actions/configure-pages@v3
      - name: Build with Jekyll
        uses: actions/jekyll-build-pages@v1
        with:
          source: ./website
          destination: ./_site
      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
+        uses: actions/upload-pages-artifact@v2

  # Deployment job
  deploy:
@@ -55,4 +55,4 @@ jobs:
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@v2
@@ -4,28 +4,32 @@ on:
  workflow_dispatch:
  push:

+env:
+  MAX_LINE_LENGTH: 110
+
 jobs:
  lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
        with:
          submodules: true
          fetch-depth: 1

      - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v1
        with:
-          python-version: "3.13"
+          python-version: 3.9
          architecture: x64

-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: true
+      - name: Install flake8
+        run: |
+          pip install flake8

-      - name: Install dependencies with uv
-        run: uv sync --all-extras --all-groups --no-sources --no-cache
-
-      - name: Run prek
-        run: uv run prek run --all-files
+      - name: Lint with flake8
+        run: |
+          cd archivebox
+          # one pass for show-stopper syntax errors or undefined names
+          flake8 . --count --show-source --statistics
+          # one pass for small stylistic things
+          flake8 . --count --max-line-length="$MAX_LINE_LENGTH" --statistics
@@ -2,64 +2,77 @@ name: Build Pip package

 on:
  workflow_dispatch:
-  workflow_call:
  push:
-    branches:
-      - '**'
-    tags:
-      - 'v*'

-env:
-  PYTHON_VERSION: "3.13"

 jobs:
  build:
+    runs-on: ubuntu-22.04
+
    permissions:
      id-token: write

-    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v4
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
        with:
-          enable-cache: true
+          submodules: true
+          fetch-depth: 1

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: '3.11'
          architecture: x64

-      - name: APT install archivebox dev + run dependencies
-        uses: awalsh128/cache-apt-pkgs-action@latest
+      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

-      - name: UV install archivebox dev + run sub-dependencies
-        run: uv sync --frozen --all-extras --no-install-project --no-install-workspace
-
-      - name: UV build archivebox and archivebox/pkgs/* packages
-        run: |
-          uv build --all
-
-      - name: Publish new package wheels and sdists to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        # ignore when publish to PyPI fails due to duplicate tag
-        continue-on-error: true
+      - uses: pdm-project/setup-pdm@v3
        with:
-          password: ${{ secrets.PYPI_PAT_SECRET }}
+          python-version: '3.11'
+          cache: true

-      - name: UV install archivebox and archivebox/pkgs/* locally for tests
-        run: uv sync --frozen --all-extras
+      - name: Install dependencies
+        run: pdm install --fail-fast --no-lock --group :all --no-self

-      - name: UV run archivebox init + archivebox version
+      - name: Build package
        run: |
-          mkdir -p data && cd data
-          uv run archivebox init \
-          && uv run archivebox version
-          # && uv run archivebox add 'https://example.com' \
-          # && uv run archivebox status \
-          # || (echo "UV Failed to run archivebox!" && exit 1)
+          pdm build
+
+      - name: Install from build
+        run: pip install ./dist/archivebox-*.whl
+
+      - name: Add some links to test
+        run: |
+          mkdir data && cd data
+          archivebox init
+          archivebox add 'https://example.com'
+          archivebox version
+          archivebox status
+
+      #- name: Publish package distributions to PyPI
+      #  run: pdm publish --no-build
+
+      # - name: Push build to PyPI
+      #   run: |
+      #     cd pip_dist/
+      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
+      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
+
+      # - name: Commit built package
+      #   run: |
+      #     cd pip_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Pip package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/pip-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: pip_dist
+
@@ -1,45 +0,0 @@
-name: Release State
-
-on:
-  push:
-    branches:
-      - '**'
-  workflow_dispatch:
-
-permissions:
-  contents: write
-  id-token: write
-
-jobs:
-  release-state:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          ref: ${{ github.ref_name }}
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: true
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Configure git identity
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-
-      - name: Run release script
-        env:
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-          GH_TOKEN: ${{ github.token }}
-          PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
-        run: ./bin/release.sh
@@ -1,42 +0,0 @@
-name: Release
-
-# Orchestrates the full release pipeline:
-#   1. Build and publish pip package to PyPI
-#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
-#   3. Build Docker images (in parallel with deb/brew)
-#
-# Individual workflows also run on push for CI (see their own triggers).
-# This workflow ensures the correct ordering during a release.
-
-on:
-  release:
-    types: [published]
-
-permissions:
-  contents: write
-  packages: write
-  id-token: write
-
-jobs:
-  pip:
-    name: Publish to PyPI
-    uses: ./.github/workflows/pip.yml
-    secrets: inherit
-
-  debian:
-    name: Build .deb packages
-    needs: pip
-    uses: ./.github/workflows/debian.yml
-    secrets: inherit
-
-  homebrew:
-    name: Update Homebrew formula
-    needs: pip
-    uses: ./.github/workflows/homebrew.yml
-    secrets: inherit
-
-  docker:
-    name: Build Docker images
-    needs: pip
-    uses: ./.github/workflows/docker.yml
-    secrets: inherit
@@ -1,158 +0,0 @@
-name: Parallel Tests
-
-on:
-  pull_request:
-    branches: [dev, main, master]
-  push:
-    branches: [dev]
-
-env:
-  PYTHONIOENCODING: utf-8
-  PYTHONLEGACYWINDOWSSTDIO: utf-8
-  USE_COLOR: False
-
-jobs:
-  discover-tests:
-    name: Discover test files
-    runs-on: ubuntu-22.04
-    outputs:
-      test-files: ${{ steps.set-matrix.outputs.test-files }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Discover test files
-        id: set-matrix
-        run: |
-          # Find all main test files
-          all_tests=$(find archivebox/tests -maxdepth 1 -name "test_*.py" -type f | sort)
-
-          # Create JSON array with test file info
-          json_array="["
-          first=true
-          for test_file in $all_tests; do
-            if [ "$first" = true ]; then
-              first=false
-            else
-              json_array+=","
-            fi
-
-            # Extract a display name for the test
-            name="main/$(basename $test_file .py | sed 's/^test_//')"
-
-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
-          done
-          json_array+="]"
-
-          echo "test-files=$json_array" >> $GITHUB_OUTPUT
-          echo "Found $(echo $all_tests | wc -w) test files"
-          echo "$json_array" | jq '.'
-
-  run-tests:
-    name: ${{ matrix.test.name }}
-    runs-on: ubuntu-22.04
-    needs: discover-tests
-
-    strategy:
-      fail-fast: false
-      matrix:
-        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
-        python: ["3.13"]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python }}
-          architecture: x64
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          version: "latest"
-
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Cache uv
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml', 'uv.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-uv-
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.1
-
-      - name: Install dependencies with uv
-        run: |
-          uv sync --dev --all-extras --no-sources
-
-      - name: Run test - ${{ matrix.test.name }}
-        run: |
-          mkdir -p tests/out
-          uv run pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
-
-  plugin-tests:
-    name: Plugin tests
-    runs-on: ubuntu-22.04
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Set up Python 3.13
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.13"
-          architecture: x64
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          version: "latest"
-
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Cache uv
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml', 'uv.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-3.13-uv-
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.1
-
-      - name: Install dependencies with uv
-        run: |
-          uv sync --dev --all-extras --no-sources
-
-      - name: Run plugin tests
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
-          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
-        run: |
-          uv run bash ./bin/test_plugins.sh --no-coverage
@@ -15,7 +15,7 @@ jobs:
      matrix:
        os: [ubuntu-22.04]
        # os: [ubuntu-22.04, macos-latest, windows-latest]
-        python: ["3.13"]
+        python: [3.11]

    steps:
      - uses: actions/checkout@v4
@@ -30,33 +30,63 @@ jobs:
          python-version: ${{ matrix.python }}
          architecture: x64

-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          version: "latest"
-
      - name: Set up Node JS
        uses: actions/setup-node@v4
        with:
-          node-version: 22
+          node-version: 20.10.0
+
+      - name: Setup PDM
+        uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true

      ### Install Python & JS Dependencies
-      - name: Cache uv
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: Cache pip
        uses: actions/cache@v3
+        id: cache-pip
        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-uv-
+            ${{ runner.os }}-${{ matrix.python }}-venv-

      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

-      - name: Install dependencies with uv
+      - name: Install pip dependencies
        run: |
-          uv sync --dev --all-extras --no-sources
+          python -m pip install --upgrade pip setuptools wheel pytest bottle build
+          python -m pip install -r requirements.txt
+          python -m pip install -e .[sonic,ldap]
+
+      - name: Get npm cache dir
+        id: npm-cache
+        run: |
+          echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
+
+      - name: Cache npm
+        uses: actions/cache@v3
+        id: cache-npm
+        with:
+          path: ${{ steps.npm-cache.outputs.dir }}
+          key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node_modules
+
+      - name: Install npm requirements
+        run: |
+          npm install
+          echo "SINGLEFILE_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/single-file" >> $GITHUB_ENV
+          echo "READABILITY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor" >> $GITHUB_ENV
+          echo "MERCURY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/mercury-parser" >> $GITHUB_ENV

      ### Run the tests
      - name: Directory listing for debugging
@@ -66,24 +96,13 @@ jobs:

      - name: Archivebox version
        run: |
-          mkdir -p tests/out/data
-          DATA_DIR="$PWD/tests/out/data" uv run archivebox version
+          archivebox version

      - name: Test built package with pytest
        # TODO: remove this exception for windows once we get tests passing on that platform
        if: ${{ !contains(matrix.os, 'windows') }}
        run: |
-          mkdir -p tests/out
-          uv run pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs
-
-      - name: Run plugin tests
-        if: ${{ !contains(matrix.os, 'windows') }}
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
-          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
-        run: |
-          uv run bash ./bin/test_plugins.sh --no-coverage
+          python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist

  docker_tests:
    runs-on: ubuntu-latest
@@ -6,26 +6,13 @@ __pycache__/
 .eggs/
 tests/out/

-# Coverage
-.coverage
-.coverage.*
-coverage.json
-coverage/
-htmlcov/
-
 # Python and Node dependencies
 venv/
 .venv/
 .docker-venv/
 node_modules/
-typings/
-
-# Ignore dev lockfiles (should always be built fresh)
-pdm.dev.lock
-requirements-dev.txt

 # Packaging artifacts
-requirements.txt
 .pdm-python
 .pdm-build
 archivebox.egg-info
@@ -34,25 +21,11 @@ build/
 dist/

 # Data folders
-lib/
-tmp/
 data/
-data*/
-archivebox/tests/data/
-archive/
+data1/
+data2/
+data3/
 output/
-logs/
-index.sqlite3
-queue.sqlite3
-*.sqlite*
-data.*
-.archivebox_id
-ArchiveBox.conf
-*.stdout
-*.stderr
-*.log
-.tmp/

 # vim
 *.sw?
-.vscode
@@ -1,3 +1,28 @@
 [submodule "docs"]
    path = docs
    url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
+
+[submodule "deb_dist"]
+    path = deb_dist
+    url = https://github.com/ArchiveBox/debian-archivebox.git
+[submodule "brew_dist"]
+	path = brew_dist
+	url = https://github.com/ArchiveBox/homebrew-archivebox.git
+[submodule "pip_dist"]
+	path = pip_dist
+	url = https://github.com/ArchiveBox/pip-archivebox.git
+[submodule "docker"]
+	path = docker
+	url = https://github.com/ArchiveBox/docker-archivebox.git
+[submodule "archivebox/vendor/base32-crockford"]
+	path = archivebox/vendor/base32-crockford
+	url = https://github.com/jbittel/base32-crockford
+[submodule "archivebox/vendor/pocket"]
+	path = archivebox/vendor/pocket
+	url = https://github.com/tapanpandita/pocket
+[submodule "archivebox/vendor/django-taggit"]
+	path = archivebox/vendor/django-taggit
+	url = https://github.com/jazzband/django-taggit
+[submodule "archivebox/vendor/python-atomicwrites"]
+	path = archivebox/vendor/python-atomicwrites
+	url = https://github.com/untitaker/python-atomicwrites
@@ -0,0 +1,19 @@
+tests/
+archivebox/
+archivebox.egg-info/
+build/
+dist/
+docs/
+etc/
+.github
+.gitmodules
+.dockerignore
+.flake8
+CNAME
+_config.yml
+docker-compose.yaml
+docker-compose.yml
+Dockerfile
+MANIFEST.in
+Pipfile
+setup.py
@@ -1,67 +0,0 @@
-default_language_version:
-  python: python3.13
-
-repos:
-  - repo: https://github.com/asottile/yesqa
-    rev: v1.5.0
-    hooks:
-      - id: yesqa
-
-  - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.1
-    hooks:
-      - id: codespell
-        additional_dependencies:
-          - tomli
-
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.20.0
-    hooks:
-      - id: pyupgrade
-        args: [--py313-plus]
-
-  - repo: https://github.com/asottile/add-trailing-comma
-    rev: v3.1.0
-    hooks:
-      - id: add-trailing-comma
-
-  - repo: local
-    hooks:
-      - id: ruff-format
-        name: ruff-format
-        entry: uv run --active ruff format
-        language: system
-        types_or: [python, pyi]
-      - id: ruff-check
-        name: ruff-check
-        entry: uv run --active ruff check --fix
-        language: system
-        types_or: [python, pyi]
-
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
-    hooks:
-      - id: check-ast
-      - id: check-toml
-      - id: check-yaml
-        exclude: ^\.github/workflows/homebrew\.yml$
-      - id: check-json
-      - id: check-merge-conflict
-      - id: check-symlinks
-      - id: destroyed-symlinks
-      - id: check-case-conflict
-      - id: check-illegal-windows-names
-      - id: check-shebang-scripts-are-executable
-        exclude: ^(archivebox/.*\.py|archivebox/tests/.*\.py|archivebox/personas/export_browser_state\.js)$
-      - id: mixed-line-ending
-      - id: fix-byte-order-marker
-      - id: end-of-file-fixer
-      - id: detect-private-key
-      - id: debug-statements
-      - id: forbid-submodules
-        exclude: ^docs$
-      - id: check-added-large-files
-        args: ["--maxkb=600"]
-      - id: name-tests-test
-        args: ["--pytest-test-first"]
-        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)
@@ -0,0 +1,34 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+submodules:
+  include: all
+  recursive: true
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    nodejs: "20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+  - pdf
+  - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: requirements.txt
+    - requirements: docs/requirements.txt
@@ -1,497 +0,0 @@
-# Claude Code Development Guide for ArchiveBox
-
-## Quick Start
-
-```bash
-# Set up dev environment (always use uv, never pip directly)
-uv sync --dev --all-extras
-
-# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
-```
-
-## Development Environment Setup
-
-### Prerequisites
- Python 3.11+ (3.13 recommended)
- uv package manager
- A non-root user for running tests (e.g., `testuser`)
-
-### Install Dependencies
-```bash
-uv sync --dev --all-extras  # Always use uv, never pip directly
-```
-
-### Activate Virtual Environment
-```bash
-source .venv/bin/activate
-```
-
-### Common Gotchas
-
-#### File Permissions
-New files created by root need permissions fixed for testuser:
-```bash
-chmod 644 archivebox/tests/test_*.py
-```
-
-#### DATA_DIR Environment Variable
-ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
-
-## Code Style Guidelines
-
-### Naming Conventions for Grep-ability
-Use consistent naming for everything to enable easy grep-ability and logical grouping:
-
-**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
-
-**Examples**:
-```python
-# Filesystem migration methods - all start with fs_
-def fs_migration_needed() -> bool: ...
-def fs_migrate() -> None: ...
-def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
-def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
-def _fs_next_version(current: str) -> str: ...
-
-# Logging methods - ALL must start with log_ or _log
-def log_migration_start(snapshot_id: str) -> None: ...
-def _log_error(message: str) -> None: ...
-def log_validation_result(ok: bool, msg: str) -> None: ...
-```
-
-**Rules**:
- Group related functions with common prefixes
- Use `_` prefix for internal/private helpers within the same family
- ALL logging-related methods MUST start with `log_` or `_log`
- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
-
-### Minimize Unique Names and Data Structures
-**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
-
-**Example - GOOD**:
-```python
-# Binary has overrides field
-binary = Binary(overrides={'TIMEOUT': '60s'})
-
-# Binary reuses the same field name and structure
-class Binary(models.Model):
-    overrides = models.JSONField(default=dict)  # Same name, same structure
-```
-
-**Example - BAD**:
-```python
-# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
-class Binary(models.Model):
-    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
-```
-
-**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
-
-## Testing
-
-### CRITICAL: Never Run as Root
-ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
-
-```bash
-# Run all migration tests
-sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
-
-# Run specific test file
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
-
-# Run single test
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
-```
-
-### Test File Structure
-```
-archivebox/tests/
-├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
-├── test_migrations_fresh.py      # Fresh install tests
-├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
-├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
-└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
-```
-
-### Test Writing Standards
-
-#### NO MOCKS - Real Tests Only
-Tests must exercise real code paths:
- Create real SQLite databases with version-specific schemas
- Seed with realistic test data
- Run actual `python -m archivebox` commands via subprocess
- Query SQLite directly to verify results
-
-**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
-
-#### NO SKIPS
-Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
-
-#### Strict Assertions
- `init` command must return exit code 0 (not `[0, 1]`)
- Verify ALL data is preserved, not just "at least one"
- Use exact counts (`==`) not loose bounds (`>=`)
-
-### Example Test Pattern
-```python
-def test_migration_preserves_snapshots(self):
-    """Migration should preserve all snapshots."""
-    result = run_archivebox(self.work_dir, ['init'], timeout=45)
-    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
-
-    ok, msg = verify_snapshot_count(self.db_path, expected_count)
-    self.assertTrue(ok, msg)
-```
-
-### Testing Gotchas
-
-#### Extractors Disabled for Speed
-Tests disable all extractors via environment variables for faster execution:
-```python
-env['SAVE_TITLE'] = 'False'
-env['SAVE_FAVICON'] = 'False'
-# ... etc
-```
-
-#### Timeout Settings
-Use appropriate timeouts for migration tests (45s for init, 60s default).
-
-### Plugin Testing & Code Coverage
-
-**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
-
-```bash
-# Run plugin tests with coverage (both Python + JavaScript)
-bash bin/test_plugins.sh screenshot
-
-# View coverage reports
-bash bin/test_plugins.sh --coverage-report
-# Or individual reports:
-coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
-```
-
-#### Plugin Test Structure
-
-Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
-
-```python
-# Correct production paths:
-# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
-# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
-
-with tempfile.TemporaryDirectory() as tmpdir:
-    data_dir = Path(tmpdir)
-
-    # Crawl-level plugin (e.g., chrome launcher)
-    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
-    chrome_dir = crawl_dir / 'chrome'
-    chrome_dir.mkdir(parents=True)
-
-    # Snapshot-level plugin (e.g., screenshot)
-    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
-    screenshot_dir = snapshot_dir / 'screenshot'
-    screenshot_dir.mkdir(parents=True)
-
-    # Run hook in its output directory
-    result = subprocess.run(
-        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
-        cwd=str(screenshot_dir),
-        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
-        capture_output=True,
-        timeout=120
-    )
-```
-
-#### Coverage Improvement Loop
-
-To improve from ~20% to 80%+:
-
-1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
-2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
-3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
-4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
-
-**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
-
-## Database Migrations
-
-### Generate and Apply Migrations
-```bash
-# Generate migrations (run from archivebox subdirectory)
-cd archivebox
-./manage.py makemigrations
-
-# Apply migrations to test database
-cd data/
-archivebox init
-```
-
-### Schema Versions
- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
-
-### Testing a Migration Path
-1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
-2. Seed with realistic test data using `seed_0_X_data()`
-3. Run `archivebox init` to trigger migrations
-4. Verify data preservation with `verify_*` functions
-5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
-
-### Squashed Migrations
-When testing 0.8.x (dev branch), you must record ALL replaced migrations:
-```python
-# The squashed migration replaces these - all must be recorded
-('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
-('core', '0024_auto_20240513_1143'),
-# ... all 52 migrations from 0023-0074 ...
-('core', '0023_new_schema'),  # Also record the squashed migration itself
-```
-
-### Migration Strategy
- Squashed migrations for clean installs
- Individual migrations recorded for upgrades from dev branch
- `replaces` attribute in squashed migrations lists what they replace
-
-### Migration Gotchas
-
-#### Circular FK References in Schemas
-SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
-
-## Plugin System Architecture
-
-### Plugin Dependency Rules
-
-Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
-However, they are allowed to depend on two shared files ONLY:
- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
-
-### Chrome-Dependent Plugins
-
-Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
-
-**Main puppeteer-based chrome installer + launcher plugin**:
- `chrome` - Core Chrome integration (CDP, launch, navigation)
-
-**Metadata extraction using chrome/chrome_utils.js / CDP**:
- `dns` - DNS resolution info
- `ssl` - SSL certificate info
- `headers` - HTTP response headers
- `redirects` - Capture redirect chains
- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
- `responses` - Capture network responses
- `consolelog` - Capture console.log output
- `title` - Extract page title
- `accessibility` - Extract accessibility tree
- `seo` - Extract SEO metadata
-
-**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
- `ublock` - uBlock Origin ad blocking
- `istilldontcareaboutcookies` - Cookie banner dismissal
- `twocaptcha` - 2captcha CAPTCHA solver integration
-
-**Page-alteration plugins to prepare the content for archiving**:
- `modalcloser` - Modal dialog dismissal
- `infiniscroll` - Infinite scroll handler
-
-**Main Extractor Outputs**:
- `dom` - DOM snapshot extraction
- `pdf` - Generate PDF snapshots
- `screenshot` - Generate screenshots
- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
-
-**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
-
-### Finding Chrome-Dependent Plugins
-
-```bash
-# Find all files containing "chrom" (case-insensitive)
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
-
-# Or get just the plugin names
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
-```
-
-**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
-
-## Architecture Notes
-
-### Crawl Model (0.9.x)
- Crawl groups multiple Snapshots from a single `add` command
- Each `add` creates one Crawl with one or more Snapshots
- Seed model was removed - crawls now store URLs directly
-
-## Code Coverage
-
-### Overview
-
-Coverage tracking is enabled for passive collection across all contexts:
- Unit tests (pytest)
- Integration tests
- Dev server (manual testing)
- CLI usage
-
-Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
-
-### Install Coverage Tools
-
-```bash
-uv sync --dev  # Installs pytest-cov and coverage
-```
-
-### Running with Coverage
-
-#### Unit Tests
-```bash
-# Run tests with coverage
-pytest --cov=archivebox --cov-report=term archivebox/tests/
-
-# Or run specific test file
-pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
-```
-
-#### Dev Server with Coverage
-```bash
-# Start dev server with coverage tracking
-coverage run --parallel-mode -m archivebox server
-
-# Or CLI commands
-coverage run --parallel-mode -m archivebox init
-coverage run --parallel-mode -m archivebox add https://example.com
-```
-
-#### Manual Testing (Always-On)
-To enable coverage during ALL Python executions (passive tracking):
-
-```bash
-# Option 1: Use coverage run wrapper
-coverage run --parallel-mode -m archivebox [command]
-
-# Option 2: Set environment variable (tracks everything)
-export COVERAGE_PROCESS_START=pyproject.toml
-# Now all Python processes will track coverage
-archivebox server
-archivebox add https://example.com
-```
-
-### Viewing Coverage
-
-#### Text Report (Quick View)
-```bash
-# Combine all parallel coverage data
-coverage combine
-
-# View summary
-coverage report
-
-# View detailed report with missing lines
-coverage report --show-missing
-
-# View specific file
-coverage report --include="archivebox/core/models.py" --show-missing
-```
-
-#### JSON Report (LLM-Friendly)
-```bash
-# Generate JSON report
-coverage json
-
-# View the JSON
-cat coverage.json | jq '.files | keys'  # List all files
-
-# Find files with low coverage
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
-
-# Find completely uncovered files (dead code candidates)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# Get missing lines for a specific file
-cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
-```
-
-#### HTML Report (Visual)
-```bash
-# Generate interactive HTML report
-coverage html
-
-# Open in browser
-open htmlcov/index.html
-```
-
-### Isolated Runs
-
-To measure coverage for specific scenarios:
-
-```bash
-# 1. Reset coverage data
-coverage erase
-
-# 2. Run your isolated test/scenario
-pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
-# OR
-coverage run --parallel-mode -m archivebox add https://example.com
-
-# 3. View results
-coverage combine
-coverage report --show-missing
-
-# 4. Optionally export for analysis
-coverage json
-```
-
-### Finding Dead Code
-
-```bash
-# 1. Run comprehensive tests + manual testing to build coverage
-pytest --cov=archivebox archivebox/tests/
-coverage run --parallel-mode -m archivebox server  # Use the app manually
-coverage combine
-
-# 2. Find files with 0% coverage (strong dead code candidates)
-coverage json
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# 3. Find files with <10% coverage (likely dead code)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
-
-# 4. Generate detailed report for analysis
-coverage report --show-missing > coverage_report.txt
-```
-
-### Tips
-
- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
- **Reset**: Use `coverage erase` to start fresh for isolated measurements
- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
-
-## Debugging Tips
-
-### Check Migration State
-```bash
-sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
-```
-
-### Check Table Schema
-```bash
-sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
-```
-
-### Verbose Test Output
-```bash
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
-```
-
-### Kill Zombie Chrome Processes
-```bash
-./bin/kill_chrome.sh
-```
@@ -1,62 +1,37 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
-#     python3.13, uv, python3-ldap
-#     curl, wget, git, dig, ping, tree, nano
-#     node, npm, single-file, readability-extractor, postlight-parser
-#     ArchiveBox, yt-dlp, playwright, chromium
+# This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
+#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file
 # Usage:
-#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
-#     docker build . -t archivebox
+#     git submodule update --init --recursive
+#     git pull --recurse-submodules
+#     docker build . -t archivebox --no-cache
 #     docker run -v "$PWD/data":/data archivebox init
 #     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
 #     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
 # Multi-arch build:
 #     docker buildx create --use
-#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
-# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+#     docker buildx build . --platform=linux/amd64,linux/arm64,linux/arm/v7 --push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
+#
+# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).


-#########################################################################################
-
-### Example: Using ArchiveBox in your own project's Dockerfile ########
-
-# FROM python:3.13-slim
-# WORKDIR /data
-# RUN pip install archivebox>=0.9.0   # use latest release here
-# RUN archivebox install
-# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
-
-#########################################################################################
-
-FROM ubuntu:24.04
+# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
+FROM python:3.11-slim-bookworm

 LABEL name="archivebox" \
    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
-    description="All-in-one self-hosted internet archiving solution" \
+    description="All-in-one personal internet archiving container" \
    homepage="https://github.com/ArchiveBox/ArchiveBox" \
-    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
-    org.opencontainers.image.title="ArchiveBox" \
-    org.opencontainers.image.vendor="ArchiveBox" \
-    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
-    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile" \
-    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
-    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
-    com.docker.desktop.extension.api.version=">= 1.4.7" \
-    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
-    com.docker.extension.publisher-url="https://archivebox.io" \
-    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
-    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
-    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
-    com.docker.extension.categories='database,utility-tools'
+    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker"

 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
+
 ######### Environment Variables #################################

-# Global build-time and runtime environment constants + default pkg manager config
+# Global system-level config
 ENV TZ=UTC \
    LANGUAGE=en_US:en \
    LC_ALL=C.UTF-8 \
@@ -68,41 +43,45 @@ ENV TZ=UTC \
    PIP_DISABLE_PIP_VERSION_CHECK=1 \
    npm_config_loglevel=error

-# Language Version config
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22
+# Version config
+ENV PYTHON_VERSION=3.11 \
+    NODE_VERSION=20

-# Non-root User config
+# User config
 ENV ARCHIVEBOX_USER="archivebox" \
    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
+    DEFAULT_PGID=911

-# ArchiveBox Source Code + Lib + Data paths
+# Global paths
 ENV CODE_DIR=/app \
    DATA_DIR=/data \
+    GLOBAL_VENV=/venv \
    PLAYWRIGHT_BROWSERS_PATH=/browsers

-# Bash SHELL config
-# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+# Application-level paths
+ENV APP_VENV=/app/.venv \
+    NODE_MODULES=/app/node_modules
+
+# Build shell config
+ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 

 ######### System Environment ####################################

-# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
-WORKDIR "$CODE_DIR"
+# Detect ArchiveBox version number by reading package.json
+COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
+RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt

-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
+# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
+RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
    && rm -f /etc/apt/apt.conf.d/docker-clean

 # Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox starting..." \
+RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
    && echo \
+    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
    && echo \
@@ -121,89 +100,34 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
+    # DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
    # https://docs.linuxserver.io/general/understanding-puid-and-pgid

 # Install system apt dependencies (adding backports to access more recent apt updates)
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
+    echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
+    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \
    && mkdir -p /etc/apt/keyrings \
    && apt-get update -qq \
-    && apt-get install -qq -y \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
        # 1. packaging dependencies
        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils \
+        zlib1g-dev dumb-init gosu cron unzip grep \
        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        tree nano iputils-ping \
        # nano iputils-ping dnsutils htop procps jq yq
    && rm -rf /var/lib/apt/lists/*

-# Install apt binary dependencies for extractors
-# COPY --from=selenium/ffmpeg:latest /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing extractor dependencies for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        git ripgrep \
-        # Packages we have also needed in the past:
-        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
-        # curl wget (already installed above)
-    && rm -rf /var/lib/apt/lists/* \
-    # Save version info
-    && ( \
-        which curl && curl --version | head -n1 \
-        && which wget && wget --version 2>&1 | head -n1 \
-        && which git && git --version 2>&1 | head -n1 \
-        # && which ffmpeg && (ffmpeg --version 2>&1 | head -n1) || true \
-        && which rg && rg --version 2>&1 | head -n1 \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Install sonic search backend
-COPY --from=archivebox/sonic:1.4.9 /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-RUN (which sonic && sonic --version) | tee -a /VERSION.txt
-
 ######### Language Environments ####################################

-# Set up Python environment
-# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
-#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
-    # && apt-get update -qq \
-    # && apt-get install -qq -y --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
-    # && rm -rf /var/lib/apt/lists/* \
-    # tell PDM to allow using global system python site packages
-    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
-    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
-    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
-    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
-    # && python3 -m venv $GLOBAL_VENV \
-    # install global dependencies / python build dependencies in GLOBAL_VENV
-    # && pip install --upgrade pip setuptools wheel \
-    # Save version info
-    # && ( \
-    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
-    #     && which pip && pip --version \
-    #     # && which pdm && pdm --version \
-    #     && echo -e '\n\n' \
-    # ) | tee -a /VERSION.txt
-
-
-# Set up Node environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing NODE $NODE_VERSION for $TARGETPLATFORM..." \
+# Install Node environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
-    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
    && apt-get update -qq \
-    && apt-get install -qq -y --no-upgrade libatomic1 \
-    && apt-get install -y --no-upgrade \
-        nodejs \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        nodejs libatomic1 python3-minimal \
    && rm -rf /var/lib/apt/lists/* \
    # Update NPM to latest version
    && npm i -g npm --cache /root/.npm \
@@ -214,183 +138,160 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
        && echo -e '\n\n' \
    ) | tee -a /VERSION.txt

-
-# Set up uv and main app /venv
-RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
-ENV UV_COMPILE_BYTECODE=1 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv
-WORKDIR "$CODE_DIR"
-# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
-    && uv venv /venv --python ${PYTHON_VERSION}
-ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
-RUN uv pip install setuptools pip \
+# Install Python environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
+    # tell PDM to allow using global system python site packages
+    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
+    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
+    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
+    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
+    # && python3 -m venv $GLOBAL_VENV \
+    # install global dependencies / python build dependencies in GLOBAL_VENV
+    # && pip install --upgrade pip setuptools wheel \
+    # Save version info
    && ( \
-        which python3 && python3 --version \
-        && which uv && uv self version \
-        && uv python find --system && uv python find \
+        which python3 && python3 --version | grep " $PYTHON_VERSION" \
+        && which pip && pip --version \
+        # && which pdm && pdm --version \
        && echo -e '\n\n' \
    ) | tee -a /VERSION.txt

+######### Extractor Dependencies ##################################

-######### ArchiveBox & Extractor Dependencies ##################################
-
-# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
+# Install apt dependencies
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing APT extractor dependencies globally using apt..." \
    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc \
-        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
-        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    && uv pip install \
-        "python-ldap>=3.4.3" \
-    && apt-get purge -y \
-        python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        curl wget git yt-dlp ffmpeg ripgrep \
+        # Packages we have also needed in the past:
+        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
+        # fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+    && rm -rf /var/lib/apt/lists/* \
+    # Save version info
+    && ( \
+        which curl && curl --version | head -n1 \
+        && which wget && wget --version 2>&1 | head -n1 \
+        && which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
+        && which git && git --version 2>&1 | head -n1 \
+        && which rg && rg --version 2>&1 | head -n1 \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt

-
-# Install apt font & rendering dependencies for chromium browser
-# TODO: figure out how much of this overlaps with `playwright install-deps chromium`
-# RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-
-# Install chromium browser binary using playwright
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing CHROMIUM dependencies, fonts, and display libraries for $TARGETPLATFORM..." \
+# Install chromium browser using playwright
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
    && apt-get update -qq \
-    && apt-get install -qq -y \
-        #fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-        #at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
-        #libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
-        #libxaw7 libxcomposite1 libxdamage1 libxfont2 \
-        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
-        # xfonts-scalable xfonts-utils xserver-common xvfb \
-        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
-        # libxss1 dbus dbus-x11 upower \
-    # && service dbus start \
-    && echo "[+] PIP Installing playwright into /venv and CHROMIUM binary into $PLAYWRIGHT_BROWSERS_PATH..." \
-    && uv pip install "playwright>=1.49.1" \
-    && uv run playwright install chromium --no-shell --with-deps \
-    && export CHROME_BINARY="$(uv run python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
+    && if [[ "$TARGETPLATFORM" == *amd64* || "$TARGETPLATFORM" == *arm64* ]]; then \
+        # install Chromium using playwright
+        pip install playwright \
+        && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
+        && playwright install --with-deps chromium \
+        && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')"; \
+    else \
+        # fall back to installing Chromium via apt-get on platforms not supported by playwright (e.g. risc, ARMv7, etc.) 
+        apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+            chromium fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+        && export CHROME_BINARY="$(which chromium)"; \
+    fi \
+    && rm -rf /var/lib/apt/lists/* \
    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
-    && ln -s /browsers/ffmpeg-*/ffmpeg-linux /usr/bin/ffmpeg \
    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \
+    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
-    # delete extra full copy of node that playwright installs (saves >100mb)
-    && rm -f /venv/lib/python$PYTHON_VERSION/site-packages/playwright/driver/node \
    # Save version info
-    && rm -rf /var/lib/apt/lists/* \
    && ( \
-        uv pip show playwright \
-        && which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
-        && which ffmpeg && ffmpeg -version \
+        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
        && echo -e '\n\n' \
    ) | tee -a /VERSION.txt

-# Install Node extractor dependencies
-ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH"
-USER $ARCHIVEBOX_USER
-WORKDIR "/home/$ARCHIVEBOX_USER/.npm"
-RUN --mount=type=cache,target=/home/archivebox/.npm_cache,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT,uid=$DEFAULT_PUID,gid=$DEFAULT_PGID \
-    echo "[+] NPM Installing node extractor dependencies into /home/$ARCHIVEBOX_USER/.npm..." \
-    && npm config set prefix "/home/$ARCHIVEBOX_USER/.npm" \
-    && npm install --global --prefer-offline --no-fund --no-audit --cache "/home/$ARCHIVEBOX_USER/.npm_cache" \
-        "@postlight/parser@^2.2.3" \
-        "readability-extractor@github:ArchiveBox/readability-extractor" \
-        "single-file-cli@^1.1.54" \
-        "puppeteer@^23.5.0" \
-        "@puppeteer/browsers@^2.4.0" \
-    && rm -Rf "/home/$ARCHIVEBOX_USER/.cache/puppeteer"
-USER root
+# Install Node dependencies
 WORKDIR "$CODE_DIR"
-RUN ( \
+COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
+RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
+    && npm ci --prefer-offline --no-audit --cache /root/.npm \
+    && ( \
        which node && node --version \
        && which npm && npm version \
-        && which postlight-parser \
-        && which readability-extractor && readability-extractor --version \
-        && which single-file && single-file --version \
-        && which puppeteer && puppeteer --version \
        && echo -e '\n\n' \
    ) | tee -a /VERSION.txt

 ######### Build Dependencies ####################################

+# Install ArchiveBox Python dependencies
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        build-essential \
+        libssl-dev libldap2-dev libsasl2-dev \
+        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
+    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
+    # && pdm use --venv in-project \
+    # && pdm run python -m ensurepip \
+    # && pdm sync --fail-fast --no-editable --group :all --no-self \
+    # && pdm export -o requirements.txt --without-hashes \
+    # && source $GLOBAL_VENV/bin/activate \
+    && pip install -r requirements.txt \
+    && apt-get purge -y \
+        build-essential \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*

-# Install ArchiveBox Python venv dependencies from uv.lock
-RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml and uv.lock..." \
-    && uv sync \
-        --frozen \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace
-    # installs the pip packages that archivebox depends on, defined in pyproject.toml and uv.lock dependencies
-
-# Install ArchiveBox Python package + workspace dependencies from source
+# Install ArchiveBox Python package from source
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && pip install \
-        --no-deps \
-        "$CODE_DIR" \
-    && ( \
-        pip show archivebox \
-        && which archivebox \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
+    # && apt-get update -qq \
+    # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
+    # && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+    #     build-essential  \
+    # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
+    && pip install -e "$CODE_DIR"[sonic,ldap] \
+    # save docker image size and always remove compilers / build tools after building is complete
+    # && apt-get purge -y build-essential \
+    # && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*

 ####################################################

 # Setup ArchiveBox runtime config
-ENV TMP_DIR=/tmp/archivebox \
-    LIB_DIR=/usr/share/archivebox/lib \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
 WORKDIR "$DATA_DIR"
-RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "$TMP_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
-    && mkdir -p "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+ENV IN_DOCKER=True
+    ## No need to set explicitly, these values will be autodetected by archivebox in docker:
+    # CHROME_SANDBOX=False \
+    # WGET_BINARY="wget" \
+    # YOUTUBEDL_BINARY="yt-dlp" \
+    # CHROME_BINARY="/usr/bin/chromium-browser" \
+    # USE_SINGLEFILE=True \
+    # SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
+    # USE_READABILITY=True \
+    # READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
+    # USE_MERCURY=True \
+    # MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"

 # Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
+RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
    ) | tee -a /VERSION.txt
-
-# Verify ArchiveBox is installed and print version info
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && gosu "$DEFAULT_PUID" archivebox version 2>&1 | tee -a /VERSION.txt || true
+RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt

 ####################################################

-# Expose ArchiveBox's main interfaces to the outside world
+# Open up the interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

-HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --silent 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
+# Optional:
+# HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
+#     CMD curl --silent 'http://localhost:8000/admin/login/' || exit 1

 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]
+CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]
@@ -1,6 +1,6 @@
 MIT License

-Copyright (c) 2024 Nick Sweeting
+Copyright (c) 2020 Nick Sweeting

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -3,4 +3,4 @@ ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E
 select = F,E9,W
 max-line-length = 130
 max-complexity = 10
-exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv,data,data*
+exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv
@@ -0,0 +1 @@
+../LICENSE
@@ -1,137 +1 @@
-#!/usr/bin/env python3
-
-# Welcome to the ArchiveBox source code! Thanks for checking it out!
-#
-# "We are swimming upstream against a great torrent of disorganization.
-# In this, our main obligation is to establish arbitrary enclaves of order and system.
-# It is the greatest possible victory to be, to continue to be, and to have been.
-# No defeat can deprive us of the success of having existed for some moment of time
-# in a universe that seems indifferent to us."
-# --Norber Weiner
-
-__package__ = "archivebox"
-
-import os
-import sys
-from pathlib import Path
-from typing import Protocol, cast
-
-from abx_plugins import get_plugins_dir
-
-
-class _ReconfigurableStream(Protocol):
-    def reconfigure(self, *, line_buffering: bool) -> object: ...
-
-
-# Force unbuffered output for real-time logs
-if hasattr(sys.stdout, "reconfigure"):
-    cast(_ReconfigurableStream, sys.stdout).reconfigure(line_buffering=True)
-    cast(_ReconfigurableStream, sys.stderr).reconfigure(line_buffering=True)
-os.environ["PYTHONUNBUFFERED"] = "1"
-
-ASCII_LOGO = """
- █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
-██╔══██╗██╔══██╗██╔════╝██║  ██║██║██║   ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
-███████║██████╔╝██║     ███████║██║██║   ██║█████╗   ██████╔╝██║   ██║ ╚███╔╝ 
-██╔══██║██╔══██╗██║     ██╔══██║██║╚██╗ ██╔╝██╔══╝   ██╔══██╗██║   ██║ ██╔██╗ 
-██║  ██║██║  ██║╚██████╗██║  ██║██║ ╚████╔╝ ███████╗ ██████╔╝╚██████╔╝██╔╝ ██╗
-╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
-"""
-
-PACKAGE_DIR = Path(__file__).resolve().parent
-
-# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models
-# # Migrations reference models like 'machine.Binary' which need to be importable
-# if str(PACKAGE_DIR) not in sys.path:
-#     sys.path.append(str(PACKAGE_DIR))
-
-os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
-os.environ["TZ"] = "UTC"
-
-# detect ArchiveBox user's UID/GID based on data dir ownership
-from .config.permissions import drop_privileges  # noqa
-
-drop_privileges()
-
-from .misc.checks import check_not_root, check_not_inside_source_dir, check_io_encoding  # noqa
-
-check_not_root()
-check_not_inside_source_dir()
-check_io_encoding()
-
-# Install monkey patches for third-party libraries
-from .misc.monkey_patches import *  # noqa
-
-# Plugin directories
-BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
-USER_PLUGINS_DIR = (
-    Path(
-        os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or os.environ.get("USER_PLUGINS_DIR") or os.environ.get("DATA_DIR", os.getcwd()),
-    )
-    / "custom_plugins"
-)
-
-# These are kept for backwards compatibility with existing code
-# that checks for plugins. The new hook system uses discover_hooks()
-ALL_PLUGINS = {
-    "builtin": BUILTIN_PLUGINS_DIR,
-    "user": USER_PLUGINS_DIR,
-}
-LOADED_PLUGINS = ALL_PLUGINS
-
-# Setup basic config, constants, paths, and version
-from .config.constants import CONSTANTS  # noqa
-from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR  # noqa
-from .config.version import VERSION  # noqa
-
-# Set MACHINE_ID env var so hook scripts can use it
-os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
-
-__version__ = VERSION
-__author__ = "ArchiveBox"
-__license__ = "MIT"
-
-ASCII_ICON = """
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-         ██                                                                            ██          
-         ██                                                                            ██        
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                    ████████████████████████████████████                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ████████████████████████████████████                    ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                 ██████████████████████████████████████████                 ██          
-         ██                 ██████████████████████████████████████████                 ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██        
-         ████████████████████████████████████████████████████████████████████████████████          
-"""
+__package__ = 'archivebox'
@@ -1,20 +1,11 @@
 #!/usr/bin/env python3
-"""This is the entrypoint for python -m archivebox ..."""

-__package__ = "archivebox"
+__package__ = 'archivebox'

-import archivebox  # noqa # make sure monkey patches are applied before anything else
 import sys

 from .cli import main

-ASCII_LOGO_MINI = r"""
-     _             _     _           ____
-    / \   _ __ ___| |__ (_)_   _____| __ )  _____  __
-   / _ \ | '__/ __| '_ \| \ \ / / _ \  _ \ / _ \ \/ /
-  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  <
- /_/   \_\_|  \___|_| |_|_| \_/ \___|____/ \___/_/\_\
-"""

-if __name__ == "__main__":
+if __name__ == '__main__':
    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1 +0,0 @@
-__package__ = "archivebox.api"
@@ -1,98 +0,0 @@
-__package__ = "archivebox.api"
-
-from django.contrib import admin
-from django.http import HttpRequest
-from signal_webhooks.admin import WebhookAdmin
-from signal_webhooks.utils import get_webhook_model
-
-from archivebox.base_models.admin import BaseModelAdmin
-
-from archivebox.api.models import APIToken
-
-
-class APITokenAdmin(BaseModelAdmin):
-    list_display = ("created_at", "id", "created_by", "token_redacted", "expires")
-    sort_fields = ("id", "created_at", "created_by", "expires")
-    readonly_fields = ("created_at", "modified_at")
-    search_fields = ("id", "created_by__username", "token")
-
-    fieldsets = (
-        (
-            "Token",
-            {
-                "fields": ("token", "expires"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Owner",
-            {
-                "fields": ("created_by",),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": ("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
-    )
-
-    list_filter = ("created_by",)
-    ordering = ["-created_at"]
-    list_per_page = 100
-
-
-class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
-    list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
-    sort_fields = ("created_at", "created_by", "id", "referenced_model", "endpoint", "last_success", "last_error")
-    readonly_fields = ("created_at", "modified_at", *WebhookAdmin.readonly_fields)
-
-    fieldsets = (
-        (
-            "Webhook",
-            {
-                "fields": ("name", "signal", "referenced_model", "endpoint"),
-                "classes": ("card", "wide"),
-            },
-        ),
-        (
-            "Authentication",
-            {
-                "fields": ("auth_token",),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Status",
-            {
-                "fields": ("enabled", "last_success", "last_error"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Owner",
-            {
-                "fields": ("created_by",),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": ("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
-    )
-
-    def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:
-        """Preserve WebhookAdmin's auth token filter with Django's current admin signature."""
-        return not lookup.startswith("auth_token") and admin.ModelAdmin.lookup_allowed(self, lookup, value, request)
-
-
-def register_admin(admin_site: admin.AdminSite) -> None:
-    admin_site.register(APIToken, APITokenAdmin)
-    admin_site.register(get_webhook_model(), CustomWebhookAdmin)
@@ -1,14 +0,0 @@
-__package__ = "archivebox.api"
-
-from django.apps import AppConfig
-
-
-class APIConfig(AppConfig):
-    name = "archivebox.api"
-    label = "api"
-
-
-def register_admin(admin_site):
-    from archivebox.api.admin import register_admin
-
-    register_admin(admin_site)
@@ -1,143 +0,0 @@
-__package__ = "archivebox.api"
-
-from datetime import timedelta
-
-from django.utils import timezone
-from django.http import HttpRequest
-from django.contrib.auth import authenticate
-from django.contrib.auth.models import User
-
-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth
-from ninja.errors import HttpError
-
-
-def get_or_create_api_token(user: User | None):
-    from archivebox.api.models import APIToken
-
-    if user and user.is_superuser:
-        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
-        if api_tokens.exists():
-            # unexpired token exists, use it
-            api_token = api_tokens.last()
-        else:
-            # does not exist, create a new one
-            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
-
-        if api_token is None:
-            return None
-        assert api_token.is_valid(), f"API token is not valid {api_token}"
-
-        return api_token
-    return None
-
-
-def auth_using_token(token: str | None, request: HttpRequest | None = None) -> User | None:
-    """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
-    from archivebox.api.models import APIToken  # lazy import model to avoid loading it at urls.py import time
-
-    user: User | None = None
-
-    submitted_empty_form = str(token).strip() in ("string", "", "None", "null")
-    if not submitted_empty_form:
-        try:
-            api_token = APIToken.objects.get(token=token)
-            if api_token.is_valid() and isinstance(api_token.created_by, User):
-                user = api_token.created_by
-                if request is not None:
-                    setattr(request, "_api_token", api_token)
-        except APIToken.DoesNotExist:
-            pass
-
-    return user
-
-
-def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
-    """Given a username and password, check if they are valid and return the corresponding user"""
-    user: User | None = None
-
-    submitted_empty_form = (username, password) in (("string", "string"), ("", ""), (None, None))
-    if not submitted_empty_form:
-        authenticated_user = authenticate(
-            username=username,
-            password=password,
-        )
-        if isinstance(authenticated_user, User):
-            user = authenticated_user
-    return user
-
-
-### Base Auth Types
-
-
-def _require_superuser(user: User | None, request: HttpRequest, auth_method: str) -> User | None:
-    if user and user.pk:
-        request.user = user
-        setattr(request, "_api_auth_method", auth_method)
-        if not user.is_superuser:
-            raise HttpError(403, "Valid credentials but User does not have permission (make sure user.is_superuser=True)")
-    return user
-
-
-### Django-Ninja-Provided Auth Methods
-
-
-class HeaderTokenAuth(APIKeyHeader):
-    """Allow authenticating by passing X-API-Key=xyz as a request header"""
-
-    param_name = "X-ArchiveBox-API-Key"
-
-    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
-        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
-
-
-class BearerTokenAuth(HttpBearer):
-    """Allow authenticating by passing Bearer=xyz as a request header"""
-
-    def authenticate(self, request: HttpRequest, token: str) -> User | None:
-        return _require_superuser(auth_using_token(token=token, request=request), request, self.__class__.__name__)
-
-
-class QueryParamTokenAuth(APIKeyQuery):
-    """Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
-
-    param_name = "api_key"
-
-    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
-        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
-
-
-class UsernameAndPasswordAuth(HttpBasicAuth):
-    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
-
-    def authenticate(self, request: HttpRequest, username: str, password: str) -> User | None:
-        return _require_superuser(
-            auth_using_password(username=username, password=password, request=request),
-            request,
-            self.__class__.__name__,
-        )
-
-
-class DjangoSessionAuth:
-    """Allow authenticating with existing Django session cookies (same-origin only)."""
-
-    def __call__(self, request: HttpRequest) -> User | None:
-        return self.authenticate(request)
-
-    def authenticate(self, request: HttpRequest, **kwargs) -> User | None:
-        user = getattr(request, "user", None)
-        if isinstance(user, User) and user.is_authenticated:
-            setattr(request, "_api_auth_method", self.__class__.__name__)
-            if not user.is_superuser:
-                raise HttpError(403, "Valid session but User does not have permission (make sure user.is_superuser=True)")
-            return user
-        return None
-
-
-### Enabled Auth Methods
-
-API_AUTH_METHODS = [
-    HeaderTokenAuth(),
-    BearerTokenAuth(),
-    QueryParamTokenAuth(),
-    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
-]
@@ -1,32 +0,0 @@
-__package__ = "archivebox.api"
-
-from django.http import HttpResponse
-
-
-class ApiCorsMiddleware:
-    """Attach permissive CORS headers for API routes (token-based auth)."""
-
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        if request.path.startswith("/api/"):
-            if request.method == "OPTIONS" and request.META.get("HTTP_ACCESS_CONTROL_REQUEST_METHOD"):
-                response = HttpResponse(status=204)
-                return self._add_cors_headers(request, response)
-
-            response = self.get_response(request)
-            return self._add_cors_headers(request, response)
-
-        return self.get_response(request)
-
-    def _add_cors_headers(self, request, response):
-        origin = request.META.get("HTTP_ORIGIN")
-        if not origin:
-            return response
-
-        response["Access-Control-Allow-Origin"] = "*"
-        response["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
-        response["Access-Control-Allow-Headers"] = "Authorization, X-ArchiveBox-API-Key, Content-Type, X-CSRFToken"
-        response["Access-Control-Max-Age"] = "600"
-        return response
@@ -1,239 +0,0 @@
-# Generated by hand on 2025-12-29
-# Creates APIToken and OutboundWebhook tables using raw SQL
-
-from django.db import migrations, models
-import django.db.models.deletion
-import django.utils.timezone
-from django.conf import settings
-from archivebox.uuid_compat import uuid7
-from archivebox.base_models.models import get_or_create_system_user_pk
-import archivebox.api.models
-import signal_webhooks.fields
-import signal_webhooks.utils
-
-
-class Migration(migrations.Migration):
-    initial = True
-
-    dependencies = [
-        ("auth", "0012_alter_user_first_name_max_length"),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.SeparateDatabaseAndState(
-            database_operations=[
-                migrations.RunSQL(
-                    sql="""
-                -- Create api_apitoken table
-                CREATE TABLE IF NOT EXISTS api_apitoken (
-                    id TEXT PRIMARY KEY NOT NULL,
-                    created_at DATETIME NOT NULL,
-                    modified_at DATETIME NOT NULL,
-
-                    token VARCHAR(32) NOT NULL UNIQUE,
-                    expires DATETIME,
-
-                    created_by_id INTEGER NOT NULL,
-
-                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
-                );
-                CREATE INDEX IF NOT EXISTS api_apitoken_created_by_id_idx ON api_apitoken(created_by_id);
-                CREATE INDEX IF NOT EXISTS api_apitoken_created_at_idx ON api_apitoken(created_at);
-                CREATE INDEX IF NOT EXISTS api_apitoken_token_idx ON api_apitoken(token);
-
-                -- Create api_outboundwebhook table
-                CREATE TABLE IF NOT EXISTS api_outboundwebhook (
-                    id TEXT PRIMARY KEY NOT NULL,
-                    created_at DATETIME NOT NULL,
-                    modified_at DATETIME NOT NULL,
-
-                    name VARCHAR(255) NOT NULL UNIQUE,
-                    signal VARCHAR(255) NOT NULL,
-                    ref VARCHAR(1024) NOT NULL,
-                    endpoint VARCHAR(2048) NOT NULL,
-                    headers TEXT NOT NULL DEFAULT '{}',
-                    auth_token TEXT NOT NULL DEFAULT '',
-                    enabled BOOLEAN NOT NULL DEFAULT 1,
-                    keep_last_response BOOLEAN NOT NULL DEFAULT 0,
-                    created DATETIME NOT NULL,
-                    updated DATETIME NOT NULL,
-                    last_response TEXT NOT NULL DEFAULT '',
-                    last_success DATETIME,
-                    last_failure DATETIME,
-
-                    created_by_id INTEGER NOT NULL,
-
-                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
-                );
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_by_id_idx ON api_outboundwebhook(created_by_id);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
-                    """,
-                    reverse_sql="""
-                DROP TABLE IF EXISTS api_outboundwebhook;
-                DROP TABLE IF EXISTS api_apitoken;
-                    """,
-                ),
-            ],
-            state_operations=[
-                migrations.CreateModel(
-                    name="APIToken",
-                    fields=[
-                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
-                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
-                        ("modified_at", models.DateTimeField(auto_now=True)),
-                        ("token", models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
-                        ("expires", models.DateTimeField(blank=True, null=True)),
-                        (
-                            "created_by",
-                            models.ForeignKey(
-                                default=get_or_create_system_user_pk,
-                                on_delete=django.db.models.deletion.CASCADE,
-                                to=settings.AUTH_USER_MODEL,
-                            ),
-                        ),
-                    ],
-                    options={
-                        "verbose_name": "API Key",
-                        "verbose_name_plural": "API Keys",
-                        "app_label": "api",
-                    },
-                ),
-                migrations.CreateModel(
-                    name="OutboundWebhook",
-                    fields=[
-                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
-                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
-                        ("modified_at", models.DateTimeField(auto_now=True)),
-                        (
-                            "name",
-                            models.CharField(db_index=True, help_text="Webhook name.", max_length=255, unique=True, verbose_name="name"),
-                        ),
-                        (
-                            "signal",
-                            models.CharField(
-                                choices=[
-                                    ("CREATE", "Create"),
-                                    ("UPDATE", "Update"),
-                                    ("DELETE", "Delete"),
-                                    ("M2M", "M2M changed"),
-                                    ("CREATE_OR_UPDATE", "Create or Update"),
-                                    ("CREATE_OR_DELETE", "Create or Delete"),
-                                    ("CREATE_OR_M2M", "Create or M2M changed"),
-                                    ("UPDATE_OR_DELETE", "Update or Delete"),
-                                    ("UPDATE_OR_M2M", "Update or M2M changed"),
-                                    ("DELETE_OR_M2M", "Delete or M2M changed"),
-                                    ("CREATE_UPDATE_OR_DELETE", "Create, Update or Delete"),
-                                    ("CREATE_UPDATE_OR_M2M", "Create, Update or M2M changed"),
-                                    ("CREATE_DELETE_OR_M2M", "Create, Delete or M2M changed"),
-                                    ("UPDATE_DELETE_OR_M2M", "Update, Delete or M2M changed"),
-                                    ("CREATE_UPDATE_DELETE_OR_M2M", "Create, Update or Delete, or M2M changed"),
-                                ],
-                                help_text="Signal the webhook fires to.",
-                                max_length=255,
-                                verbose_name="signal",
-                            ),
-                        ),
-                        (
-                            "ref",
-                            models.CharField(
-                                db_index=True,
-                                help_text="Dot import notation to the model the webhook is for.",
-                                max_length=1023,
-                                validators=[signal_webhooks.utils.model_from_reference],
-                                verbose_name="referenced model",
-                            ),
-                        ),
-                        (
-                            "endpoint",
-                            models.URLField(help_text="Target endpoint for this webhook.", max_length=2047, verbose_name="endpoint"),
-                        ),
-                        (
-                            "headers",
-                            models.JSONField(
-                                blank=True,
-                                default=dict,
-                                help_text="Headers to send with the webhook request.",
-                                validators=[signal_webhooks.utils.is_dict],
-                                verbose_name="headers",
-                            ),
-                        ),
-                        (
-                            "auth_token",
-                            signal_webhooks.fields.TokenField(
-                                blank=True,
-                                default="",
-                                help_text="Authentication token to use in an Authorization header.",
-                                max_length=8000,
-                                validators=[signal_webhooks.utils.decode_cipher_key],
-                                verbose_name="authentication token",
-                            ),
-                        ),
-                        ("enabled", models.BooleanField(default=True, help_text="Is this webhook enabled?", verbose_name="enabled")),
-                        (
-                            "keep_last_response",
-                            models.BooleanField(
-                                default=False,
-                                help_text="Should the webhook keep a log of the latest response it got?",
-                                verbose_name="keep last response",
-                            ),
-                        ),
-                        (
-                            "created",
-                            models.DateTimeField(auto_now_add=True, help_text="When the webhook was created.", verbose_name="created"),
-                        ),
-                        (
-                            "updated",
-                            models.DateTimeField(auto_now=True, help_text="When the webhook was last updated.", verbose_name="updated"),
-                        ),
-                        (
-                            "last_response",
-                            models.CharField(
-                                blank=True,
-                                default="",
-                                help_text="Latest response to this webhook.",
-                                max_length=8000,
-                                verbose_name="last response",
-                            ),
-                        ),
-                        (
-                            "last_success",
-                            models.DateTimeField(
-                                default=None,
-                                help_text="When the webhook last succeeded.",
-                                null=True,
-                                verbose_name="last success",
-                            ),
-                        ),
-                        (
-                            "last_failure",
-                            models.DateTimeField(
-                                default=None,
-                                help_text="When the webhook last failed.",
-                                null=True,
-                                verbose_name="last failure",
-                            ),
-                        ),
-                        (
-                            "created_by",
-                            models.ForeignKey(
-                                default=get_or_create_system_user_pk,
-                                on_delete=django.db.models.deletion.CASCADE,
-                                to=settings.AUTH_USER_MODEL,
-                            ),
-                        ),
-                    ],
-                    options={
-                        "verbose_name": "API Outbound Webhook",
-                        "app_label": "api",
-                    },
-                ),
-                migrations.AddConstraint(
-                    model_name="outboundwebhook",
-                    constraint=models.UniqueConstraint(fields=["ref", "endpoint"], name="prevent_duplicate_hooks_api_outboundwebhook"),
-                ),
-            ],
-        ),
-    ]
@@ -1,54 +0,0 @@
-__package__ = "archivebox.api"
-
-import secrets
-from archivebox.uuid_compat import uuid7
-
-from django.conf import settings
-from django.db import models
-from django.utils import timezone
-from django_stubs_ext.db.models import TypedModelMeta
-from signal_webhooks.models import WebhookBase
-
-from archivebox.base_models.models import get_or_create_system_user_pk
-
-
-def generate_secret_token() -> str:
-    return secrets.token_hex(16)
-
-
-class APIToken(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-    token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
-    expires = models.DateTimeField(null=True, blank=True)
-
-    class Meta(TypedModelMeta):
-        app_label = "api"
-        verbose_name = "API Key"
-        verbose_name_plural = "API Keys"
-
-    def __str__(self) -> str:
-        return self.token
-
-    @property
-    def token_redacted(self):
-        return f"************{self.token[-4:]}"
-
-    def is_valid(self, for_date=None):
-        return not self.expires or self.expires >= (for_date or timezone.now())
-
-
-class OutboundWebhook(WebhookBase):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-
-    class Meta(WebhookBase.Meta):
-        app_label = "api"
-        verbose_name = "API Outbound Webhook"
-
-    def __str__(self) -> str:
-        return f"[{self.id}] {self.ref} -> {self.endpoint}"
@@ -1,16 +0,0 @@
-__package__ = "archivebox.api"
-
-from django.urls import path
-from django.views.generic.base import RedirectView
-
-from .v1_api import urls as v1_api_urls
-
-urlpatterns = [
-    path("", RedirectView.as_view(url="/api/v1/docs")),
-    path("v1/", RedirectView.as_view(url="/api/v1/docs")),
-    path("v1/", v1_api_urls),
-    path("v1", RedirectView.as_view(url="/api/v1/docs")),
-    # ... v2 can be added here ...
-    # path("v2/",              v2_api_urls),
-    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
-]
@@ -1,135 +0,0 @@
-__package__ = "archivebox.api"
-
-
-from io import StringIO
-from traceback import format_exception
-from contextlib import redirect_stdout, redirect_stderr
-
-from django.http import HttpRequest, HttpResponse
-from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
-from django.contrib.auth.models import User
-
-from ninja import NinjaAPI, Swagger
-
-# TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
-
-from archivebox.config import VERSION
-from archivebox.config.version import get_COMMIT_HASH
-
-from archivebox.api.auth import API_AUTH_METHODS
-from archivebox.api.models import APIToken
-
-
-COMMIT_HASH = get_COMMIT_HASH() or "unknown"
-
-html_description = f"""
-<h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
-<br/>
-<i><b>WARNING: This API is still in an early development stage and may change!</b></i>
-<br/>
-<ul>
-<li>⬅️ Manage your server: <a href="/admin/api/"><b>Setup API Keys</b></a>, <a href="/admin/">Go to your Server Admin UI</a>, <a href="/">Go to your Snapshots list</a> 
-<li>💬 Ask questions and get help here: <a href="https://zulip.archivebox.io">ArchiveBox Chat Forum</a></li>
-<li>🐞 Report API bugs here: <a href="https://github.com/ArchiveBox/ArchiveBox/issues">Github Issues</a></li>
-<li>📚 ArchiveBox Documentation: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Github Wiki</a></li>
-<li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
-</ul>
-<small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
-"""
-
-
-def register_urls(api: NinjaAPI) -> NinjaAPI:
-    api.add_router("/auth/", "archivebox.api.v1_auth.router")
-    api.add_router("/core/", "archivebox.api.v1_core.router")
-    api.add_router("/crawls/", "archivebox.api.v1_crawls.router")
-    api.add_router("/cli/", "archivebox.api.v1_cli.router")
-    api.add_router("/machine/", "archivebox.api.v1_machine.router")
-    return api
-
-
-class NinjaAPIWithIOCapture(NinjaAPI):
-    def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
-        stdout, stderr = StringIO(), StringIO()
-
-        with redirect_stderr(stderr):
-            with redirect_stdout(stdout):
-                setattr(request, "stdout", stdout)
-                setattr(request, "stderr", stderr)
-
-                response = super().create_temporal_response(request)
-
-        # Disable caching of API responses entirely
-        response["Cache-Control"] = "no-store"
-
-        # Add debug stdout and stderr headers to response
-        response["X-ArchiveBox-Stdout"] = stdout.getvalue().replace("\n", "\\n")[:200]
-        response["X-ArchiveBox-Stderr"] = stderr.getvalue().replace("\n", "\\n")[:200]
-        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
-
-        # Add Auth Headers to response
-        api_token_attr = getattr(request, "_api_token", None)
-        api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
-        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
-
-        response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
-        response["X-ArchiveBox-Auth-Expires"] = token_expiry
-        response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
-        response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
-
-        # import ipdb; ipdb.set_trace()
-        # print('RESPONDING NOW', response)
-
-        return response
-
-
-api = NinjaAPIWithIOCapture(
-    title="ArchiveBox API",
-    description=html_description,
-    version=VERSION,
-    auth=API_AUTH_METHODS,
-    urls_namespace="api-1",
-    docs=Swagger(settings={"persistAuthorization": True}),
-    # docs_decorator=login_required,
-    # renderer=ORJSONRenderer(),
-)
-api = register_urls(api)
-urls = api.urls
-
-
-@api.exception_handler(Exception)
-def generic_exception_handler(request, err):
-    status = 503
-    if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
-        status = 404
-
-    print("".join(format_exception(err)))
-
-    return api.create_response(
-        request,
-        {
-            "succeeded": False,
-            "message": f"{err.__class__.__name__}: {err}",
-            "errors": [
-                "".join(format_exception(err)),
-                # or send simpler parent-only traceback:
-                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
-            ],
-        },
-        status=status,
-    )
-
-
-# import orjson
-# from ninja.renderers import BaseRenderer
-# class ORJSONRenderer(BaseRenderer):
-#     media_type = "application/json"
-#     def render(self, request, data, *, response_status):
-#         return {
-#             "success": True,
-#             "errors": [],
-#             "result": data,
-#             "stdout": ansi_to_html(stdout.getvalue().strip()),
-#             "stderr": ansi_to_html(stderr.getvalue().strip()),
-#         }
-#         return orjson.dumps(data)
@@ -1,65 +0,0 @@
-__package__ = "archivebox.api"
-
-from django.http import HttpRequest
-
-from ninja import Router, Schema
-
-from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token
-
-
-router = Router(tags=["Authentication"], auth=None)
-
-
-class PasswordAuthSchema(Schema):
-    """Schema for a /get_api_token request"""
-
-    username: str | None = None
-    password: str | None = None
-
-
-@router.post(
-    "/get_api_token",
-    auth=None,
-    summary="Generate an API token for a given username & password (or currently logged-in user)",
-)  # auth=None because they are not authed yet
-def get_api_token(request: HttpRequest, auth_data: PasswordAuthSchema):
-    user = auth_using_password(
-        username=auth_data.username,
-        password=auth_data.password,
-        request=request,
-    )
-
-    if user and user.is_superuser:
-        api_token = get_or_create_api_token(user)
-        assert api_token is not None, "Failed to create API token"
-        return {
-            "success": True,
-            "user_id": str(user.pk),
-            "username": user.username,
-            "token": api_token.token,
-            "expires": api_token.expires.isoformat() if api_token.expires else None,
-        }
-
-    return {"success": False, "errors": ["Invalid credentials"]}
-
-
-class TokenAuthSchema(Schema):
-    """Schema for a /check_api_token request"""
-
-    token: str
-
-
-@router.post(
-    "/check_api_token",
-    auth=None,
-    summary="Validate an API token to make sure its valid and non-expired",
-)  # auth=None because they are not authed yet
-def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
-    user = auth_using_token(
-        token=token_data.token,
-        request=request,
-    )
-    if user:
-        return {"success": True, "user_id": str(user.pk)}
-
-    return {"success": False, "user_id": None}
@@ -1,284 +0,0 @@
-__package__ = "archivebox.api"
-
-import json
-from io import StringIO
-from typing import Any
-from enum import Enum
-
-from django.http import HttpRequest
-
-from ninja import Router, Schema
-
-from archivebox.misc.util import ansi_to_html
-from archivebox.config.common import ARCHIVING_CONFIG
-
-
-# from .auth import API_AUTH_METHODS
-
-# router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=["ArchiveBox CLI Sub-Commands"])
-
-
-# Schemas
-
-JSONType = list[Any] | dict[str, Any] | bool | int | str | None
-
-
-class CLICommandResponseSchema(Schema):
-    success: bool
-    errors: list[str]
-    result: JSONType
-    result_format: str = "str"
-    stdout: str
-    stderr: str
-
-
-class FilterTypeChoices(str, Enum):
-    exact = "exact"
-    substring = "substring"
-    regex = "regex"
-    domain = "domain"
-    tag = "tag"
-    timestamp = "timestamp"
-
-
-class StatusChoices(str, Enum):
-    indexed = "indexed"
-    archived = "archived"
-    unarchived = "unarchived"
-    present = "present"
-    valid = "valid"
-    invalid = "invalid"
-    duplicate = "duplicate"
-    orphaned = "orphaned"
-    corrupted = "corrupted"
-    unrecognized = "unrecognized"
-
-
-class AddCommandSchema(Schema):
-    urls: list[str]
-    tag: str = ""
-    depth: int = 0
-    parser: str = "auto"
-    plugins: str = ""
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW  # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW
-    overwrite: bool = False
-    index_only: bool = False
-
-
-class UpdateCommandSchema(Schema):
-    resume: str | None = None
-    after: float | None = 0
-    before: float | None = 999999999999999
-    filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = ["https://example.com"]
-    batch_size: int = 100
-    continuous: bool = False
-
-
-class ScheduleCommandSchema(Schema):
-    import_path: str | None = None
-    add: bool = False
-    show: bool = False
-    foreground: bool = False
-    run_all: bool = False
-    quiet: bool = False
-    every: str | None = None
-    tag: str = ""
-    depth: int = 0
-    overwrite: bool = False
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW
-    clear: bool = False
-
-
-class ListCommandSchema(Schema):
-    filter_patterns: list[str] | None = ["https://example.com"]
-    filter_type: str = FilterTypeChoices.substring
-    status: StatusChoices = StatusChoices.indexed
-    after: float | None = 0
-    before: float | None = 999999999999999
-    sort: str = "bookmarked_at"
-    as_json: bool = True
-    as_html: bool = False
-    as_csv: str | None = "timestamp,url"
-    with_headers: bool = False
-
-
-class RemoveCommandSchema(Schema):
-    delete: bool = True
-    after: float | None = 0
-    before: float | None = 999999999999999
-    filter_type: str = FilterTypeChoices.exact
-    filter_patterns: list[str] | None = ["https://example.com"]
-
-
-@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
-def cli_add(request: HttpRequest, args: AddCommandSchema):
-    from archivebox.cli.archivebox_add import add
-
-    crawl, snapshots = add(
-        urls=args.urls,
-        tag=args.tag,
-        depth=args.depth,
-        update=args.update,
-        index_only=args.index_only,
-        overwrite=args.overwrite,
-        plugins=args.plugins,
-        parser=args.parser,
-        bg=True,  # Always run in background for API calls
-        created_by_id=request.user.pk,
-    )
-
-    snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
-    result_payload = {
-        "crawl_id": str(crawl.id),
-        "num_snapshots": len(snapshot_ids),
-        "snapshot_ids": snapshot_ids,
-        "queued_urls": args.urls,
-    }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
-
-    return {
-        "success": True,
-        "errors": [],
-        "result": result_payload,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
-def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import update
-
-    result = update(
-        filter_patterns=args.filter_patterns or [],
-        filter_type=args.filter_type or FilterTypeChoices.substring,
-        after=args.after,
-        before=args.before,
-        resume=args.resume,
-        batch_size=args.batch_size,
-        continuous=args.continuous,
-    )
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
-def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
-    from archivebox.cli.archivebox_schedule import schedule
-
-    result = schedule(
-        import_path=args.import_path,
-        add=args.add,
-        show=args.show,
-        foreground=args.foreground,
-        run_all=args.run_all,
-        quiet=args.quiet,
-        clear=args.clear,
-        every=args.every,
-        tag=args.tag,
-        depth=args.depth,
-        overwrite=args.overwrite,
-        update=args.update,
-    )
-
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
-def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_search import search
-
-    result = search(
-        filter_patterns=args.filter_patterns,
-        filter_type=args.filter_type,
-        status=args.status,
-        after=args.after,
-        before=args.before,
-        sort=args.sort,
-        csv=args.as_csv,
-        json=args.as_json,
-        html=args.as_html,
-        with_headers=args.with_headers,
-    )
-
-    result_format = "txt"
-    if args.as_json:
-        result_format = "json"
-        result = json.loads(result)
-    elif args.as_html:
-        result_format = "html"
-    elif args.as_csv:
-        result_format = "csv"
-
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "result_format": result_format,
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
-def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
-    from archivebox.cli.archivebox_remove import remove
-    from archivebox.cli.archivebox_search import get_snapshots
-    from archivebox.core.models import Snapshot
-
-    filter_patterns = args.filter_patterns or []
-    snapshots_to_remove = get_snapshots(
-        filter_patterns=filter_patterns,
-        filter_type=args.filter_type,
-        after=args.after,
-        before=args.before,
-    )
-    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
-
-    remove(
-        yes=True,  # no way to interactively ask for confirmation via API, so we force yes
-        delete=args.delete,
-        snapshots=snapshots_to_remove,
-        before=args.before,
-        after=args.after,
-        filter_type=args.filter_type,
-        filter_patterns=filter_patterns,
-    )
-
-    result = {
-        "removed_count": len(removed_snapshot_ids),
-        "removed_snapshot_ids": removed_snapshot_ids,
-        "remaining_snapshots": Snapshot.objects.count(),
-    }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
@@ -1,829 +0,0 @@
-__package__ = "archivebox.api"
-
-import math
-from collections import defaultdict
-from uuid import UUID
-from typing import Union, Any, Annotated
-from datetime import datetime
-
-from django.db.models import Model, Q, Sum
-from django.db.models.functions import Coalesce
-from django.conf import settings
-from django.http import HttpRequest, HttpResponse
-from django.core.exceptions import ValidationError
-from django.contrib.auth import get_user_model
-from django.contrib.auth.models import User
-from django.shortcuts import redirect
-from django.utils import timezone
-
-from ninja import Router, Schema, FilterLookup, FilterSchema, Query
-from ninja.pagination import paginate, PaginationBase
-from ninja.errors import HttpError
-
-from archivebox.core.models import Snapshot, ArchiveResult, Tag
-from archivebox.api.auth import auth_using_token
-from archivebox.config.common import SERVER_CONFIG
-from archivebox.core.tag_utils import (
-    build_tag_cards,
-    delete_tag as delete_tag_record,
-    export_tag_snapshots_jsonl,
-    export_tag_urls,
-    get_matching_tags,
-    get_or_create_tag,
-    get_tag_by_ref,
-    normalize_created_by_filter,
-    normalize_created_year_filter,
-    normalize_has_snapshots_filter,
-    normalize_tag_sort,
-    rename_tag as rename_tag_record,
-)
-from archivebox.crawls.models import Crawl
-from archivebox.api.v1_crawls import CrawlSchema
-
-
-router = Router(tags=["Core Models"])
-
-
-class CustomPagination(PaginationBase):
-    class Input(PaginationBase.Input):
-        limit: int = 200
-        offset: int = 0
-        page: int = 0
-
-    class Output(PaginationBase.Output):
-        count: int
-        total_items: int
-        total_pages: int
-        page: int
-        limit: int
-        offset: int
-        num_items: int
-        items: list[Any]
-
-    def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
-        limit = min(pagination.limit, 500)
-        offset = pagination.offset or (pagination.page * limit)
-        total = queryset.count()
-        total_pages = math.ceil(total / limit)
-        current_page = math.ceil(offset / (limit + 1))
-        items = queryset[offset : offset + limit]
-        return {
-            "count": total,
-            "total_items": total,
-            "total_pages": total_pages,
-            "page": current_page,
-            "limit": limit,
-            "offset": offset,
-            "num_items": len(items),
-            "items": items,
-        }
-
-
-### ArchiveResult #########################################################################
-
-
-class MinimalArchiveResultSchema(Schema):
-    TYPE: str = "core.models.ArchiveResult"
-    id: UUID
-    created_at: datetime | None
-    modified_at: datetime | None
-    created_by_id: str
-    created_by_username: str
-    status: str
-    retry_at: datetime | None = None
-    plugin: str
-    hook_name: str
-    process_id: UUID | None
-    cmd_version: str | None
-    cmd: list[str] | None
-    pwd: str | None
-    output_str: str
-    output_json: dict[str, Any] | None
-    output_files: dict[str, dict[str, Any]] | None
-    output_size: int
-    output_mimetypes: str
-    start_ts: datetime | None
-    end_ts: datetime | None
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by.pk)
-
-    @staticmethod
-    def resolve_created_by_username(obj) -> str:
-        return obj.created_by.username
-
-    @staticmethod
-    def resolve_output_files(obj):
-        return obj.output_file_map()
-
-    @staticmethod
-    def resolve_output_mimetypes(obj) -> str:
-        mime_sizes: dict[str, int] = defaultdict(int)
-        for metadata in obj.output_file_map().values():
-            if not isinstance(metadata, dict):
-                continue
-            mimetype = str(metadata.get("mimetype") or "").strip()
-            try:
-                size = max(int(metadata.get("size") or 0), 0)
-            except (TypeError, ValueError):
-                size = 0
-            if mimetype and size:
-                mime_sizes[mimetype] += size
-        if mime_sizes:
-            return ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-        return obj.output_mimetypes or ""
-
-
-class ArchiveResultSchema(MinimalArchiveResultSchema):
-    TYPE: str = "core.models.ArchiveResult"
-    snapshot_id: UUID
-    snapshot_timestamp: str
-    snapshot_url: str
-    snapshot_tags: list[str]
-
-    @staticmethod
-    def resolve_snapshot_timestamp(obj):
-        return obj.snapshot.timestamp
-
-    @staticmethod
-    def resolve_snapshot_url(obj):
-        return obj.snapshot.url
-
-    @staticmethod
-    def resolve_snapshot_id(obj):
-        return obj.snapshot_id
-
-    @staticmethod
-    def resolve_snapshot_tags(obj):
-        return sorted(tag.name for tag in obj.snapshot.tags.all())
-
-
-class ArchiveResultFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup(["id__startswith", "snapshot__id__startswith", "snapshot__timestamp__startswith"])] = None
-    search: Annotated[
-        str | None,
-        FilterLookup(
-            [
-                "snapshot__url__icontains",
-                "snapshot__title__icontains",
-                "snapshot__tags__name__icontains",
-                "plugin",
-                "output_str__icontains",
-                "id__startswith",
-                "snapshot__id__startswith",
-                "snapshot__timestamp__startswith",
-            ],
-        ),
-    ] = None
-    snapshot_id: Annotated[str | None, FilterLookup(["snapshot__id__startswith", "snapshot__timestamp__startswith"])] = None
-    snapshot_url: Annotated[str | None, FilterLookup("snapshot__url__icontains")] = None
-    snapshot_tag: Annotated[str | None, FilterLookup("snapshot__tags__name__icontains")] = None
-    status: Annotated[str | None, FilterLookup("status")] = None
-    output_str: Annotated[str | None, FilterLookup("output_str__icontains")] = None
-    plugin: Annotated[str | None, FilterLookup("plugin__icontains")] = None
-    hook_name: Annotated[str | None, FilterLookup("hook_name__icontains")] = None
-    process_id: Annotated[str | None, FilterLookup("process__id__startswith")] = None
-    cmd: Annotated[str | None, FilterLookup("cmd__0__icontains")] = None
-    pwd: Annotated[str | None, FilterLookup("pwd__icontains")] = None
-    cmd_version: Annotated[str | None, FilterLookup("cmd_version")] = None
-    created_at: Annotated[datetime | None, FilterLookup("created_at")] = None
-    created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
-    created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
-
-
-@router.get("/archiveresults", response=list[ArchiveResultSchema], url_name="get_archiveresult")
-@paginate(CustomPagination)
-def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
-    """List all ArchiveResult entries matching these filters."""
-    return filters.filter(ArchiveResult.objects.all()).distinct()
-
-
-@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
-def get_archiveresult(request: HttpRequest, archiveresult_id: str):
-    """Get a specific ArchiveResult by id."""
-    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id))
-
-
-### Snapshot #########################################################################
-
-
-class SnapshotSchema(Schema):
-    TYPE: str = "core.models.Snapshot"
-    id: UUID
-    created_by_id: str
-    created_by_username: str
-    created_at: datetime
-    modified_at: datetime
-    status: str
-    retry_at: datetime | None
-    bookmarked_at: datetime
-    downloaded_at: datetime | None
-    url: str
-    tags: list[str]
-    title: str | None
-    timestamp: str
-    archive_path: str
-    archive_size: int
-    output_size: int
-    num_archiveresults: int
-    archiveresults: list[MinimalArchiveResultSchema]
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by.pk)
-
-    @staticmethod
-    def resolve_created_by_username(obj):
-        return obj.created_by.username
-
-    @staticmethod
-    def resolve_tags(obj):
-        return sorted(tag.name for tag in obj.tags.all())
-
-    @staticmethod
-    def resolve_archive_size(obj):
-        return int(getattr(obj, "output_size_sum", obj.archive_size) or 0)
-
-    @staticmethod
-    def resolve_output_size(obj):
-        return SnapshotSchema.resolve_archive_size(obj)
-
-    @staticmethod
-    def resolve_num_archiveresults(obj, context):
-        return obj.archiveresult_set.all().distinct().count()
-
-    @staticmethod
-    def resolve_archiveresults(obj, context):
-        if bool(getattr(context["request"], "with_archiveresults", False)):
-            return obj.archiveresult_set.all().distinct()
-        return ArchiveResult.objects.none()
-
-
-class SnapshotUpdateSchema(Schema):
-    status: str | None = None
-    retry_at: datetime | None = None
-    tags: list[str] | None = None
-
-
-class SnapshotCreateSchema(Schema):
-    url: str
-    crawl_id: str | None = None
-    depth: int = 0
-    title: str | None = None
-    tags: list[str] | None = None
-    status: str | None = None
-
-
-class SnapshotDeleteResponseSchema(Schema):
-    success: bool
-    snapshot_id: str
-    crawl_id: str
-    deleted_count: int
-
-
-def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
-    return [tag.strip() for tag in (tags or []) if tag and tag.strip()]
-
-
-class SnapshotFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup(["id__icontains", "timestamp__startswith"])] = None
-    created_by_id: Annotated[str | None, FilterLookup("crawl__created_by_id")] = None
-    created_by_username: Annotated[str | None, FilterLookup("crawl__created_by__username__icontains")] = None
-    created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
-    created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
-    created_at: Annotated[datetime | None, FilterLookup("created_at")] = None
-    modified_at: Annotated[datetime | None, FilterLookup("modified_at")] = None
-    modified_at__gte: Annotated[datetime | None, FilterLookup("modified_at__gte")] = None
-    modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
-    search: Annotated[
-        str | None,
-        FilterLookup(["url__icontains", "title__icontains", "tags__name__icontains", "id__icontains", "timestamp__startswith"]),
-    ] = None
-    url: Annotated[str | None, FilterLookup("url")] = None
-    tag: Annotated[str | None, FilterLookup("tags__name")] = None
-    title: Annotated[str | None, FilterLookup("title__icontains")] = None
-    timestamp: Annotated[str | None, FilterLookup("timestamp__startswith")] = None
-    bookmarked_at__gte: Annotated[datetime | None, FilterLookup("bookmarked_at__gte")] = None
-    bookmarked_at__lt: Annotated[datetime | None, FilterLookup("bookmarked_at__lt")] = None
-
-
-@router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
-@paginate(CustomPagination)
-def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
-    """List all Snapshot entries matching these filters."""
-    setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0))
-    return filters.filter(queryset).distinct()
-
-
-@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
-def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
-    """Get a specific Snapshot by id."""
-    setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0))
-    try:
-        return queryset.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        return queryset.get(Q(id__icontains=snapshot_id))
-
-
-@router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
-def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
-    tags = normalize_tag_list(data.tags)
-    if data.status is not None and data.status not in Snapshot.StatusChoices.values:
-        raise HttpError(400, f"Invalid status: {data.status}")
-    if not data.url.strip():
-        raise HttpError(400, "URL is required")
-    if data.depth not in (0, 1, 2, 3, 4):
-        raise HttpError(400, "depth must be between 0 and 4")
-
-    if data.crawl_id:
-        crawl = Crawl.objects.get(id__icontains=data.crawl_id)
-        crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
-        tags = tags or crawl_tags
-    else:
-        crawl = Crawl.objects.create(
-            urls=data.url,
-            max_depth=max(data.depth, 0),
-            tags_str=",".join(tags),
-            status=Crawl.StatusChoices.QUEUED,
-            retry_at=timezone.now(),
-            created_by=request.user if isinstance(request.user, User) else None,
-        )
-
-    snapshot_defaults = {
-        "depth": data.depth,
-        "title": data.title,
-        "timestamp": str(timezone.now().timestamp()),
-        "status": data.status or Snapshot.StatusChoices.QUEUED,
-        "retry_at": timezone.now(),
-    }
-    snapshot, _ = Snapshot.objects.get_or_create(
-        url=data.url,
-        crawl=crawl,
-        defaults=snapshot_defaults,
-    )
-
-    update_fields: list[str] = []
-    if data.title is not None and snapshot.title != data.title:
-        snapshot.title = data.title
-        update_fields.append("title")
-    if data.status is not None and snapshot.status != data.status:
-        if data.status not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {data.status}")
-        snapshot.status = data.status
-        update_fields.append("status")
-    if update_fields:
-        update_fields.append("modified_at")
-        snapshot.save(update_fields=update_fields)
-
-    if tags:
-        snapshot.save_tags(tags)
-
-    try:
-        snapshot.ensure_crawl_symlink()
-    except Exception:
-        pass
-
-    setattr(request, "with_archiveresults", False)
-    return snapshot
-
-
-@router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
-def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
-    """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    try:
-        snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        snapshot = Snapshot.objects.get(Q(id__icontains=snapshot_id))
-
-    payload = data.dict(exclude_unset=True)
-    update_fields = ["modified_at"]
-    tags = payload.pop("tags", None)
-
-    if "status" in payload:
-        if payload["status"] not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        snapshot.status = payload["status"]
-        if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
-            snapshot.retry_at = None
-        update_fields.append("status")
-
-    if "retry_at" in payload:
-        snapshot.retry_at = payload["retry_at"]
-        update_fields.append("retry_at")
-
-    if tags is not None:
-        snapshot.save_tags(normalize_tag_list(tags))
-
-    snapshot.save(update_fields=update_fields)
-    setattr(request, "with_archiveresults", False)
-    return snapshot
-
-
-@router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
-def delete_snapshot(request: HttpRequest, snapshot_id: str):
-    snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
-    snapshot_id_str = str(snapshot.id)
-    crawl_id_str = str(snapshot.crawl.pk)
-    deleted_count, _ = snapshot.delete()
-    return {
-        "success": True,
-        "snapshot_id": snapshot_id_str,
-        "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
-    }
-
-
-### Tag #########################################################################
-
-
-class TagSchema(Schema):
-    TYPE: str = "core.models.Tag"
-    id: int
-    modified_at: datetime
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-    name: str
-    slug: str
-    num_snapshots: int
-    snapshots: list[SnapshotSchema]
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-
-    @staticmethod
-    def resolve_created_by_username(obj):
-        user_model = get_user_model()
-        user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
-        return username if isinstance(username, str) else str(user)
-
-    @staticmethod
-    def resolve_num_snapshots(obj, context):
-        return obj.snapshot_set.all().distinct().count()
-
-    @staticmethod
-    def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
-            return obj.snapshot_set.all().distinct()
-        return Snapshot.objects.none()
-
-
-@router.get("/tags", response=list[TagSchema], url_name="get_tags")
-@paginate(CustomPagination)
-def get_tags(request: HttpRequest):
-    setattr(request, "with_snapshots", False)
-    setattr(request, "with_archiveresults", False)
-    return get_matching_tags()
-
-
-@router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
-def get_tag(request: HttpRequest, tag_id: str, with_snapshots: bool = True):
-    setattr(request, "with_snapshots", with_snapshots)
-    setattr(request, "with_archiveresults", False)
-    try:
-        return get_tag_by_ref(tag_id)
-    except (Tag.DoesNotExist, ValidationError):
-        raise HttpError(404, "Tag not found")
-
-
-@router.get(
-    "/any/{id}",
-    response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema, CrawlSchema],
-    url_name="get_any",
-    summary="Get any object by its ID",
-)
-def get_any(request: HttpRequest, id: str):
-    """Get any object by its ID (e.g. snapshot, archiveresult, tag, crawl, etc.)."""
-    setattr(request, "with_snapshots", False)
-    setattr(request, "with_archiveresults", False)
-
-    for getter in [get_snapshot, get_archiveresult, get_tag]:
-        try:
-            response = getter(request, id)
-            if isinstance(response, Model):
-                return redirect(
-                    f"/api/v1/{response._meta.app_label}/{response._meta.model_name}/{response.pk}?{request.META['QUERY_STRING']}",
-                )
-        except Exception:
-            pass
-
-    try:
-        from archivebox.api.v1_crawls import get_crawl
-
-        response = get_crawl(request, id)
-        if isinstance(response, Model):
-            return redirect(f"/api/v1/{response._meta.app_label}/{response._meta.model_name}/{response.pk}?{request.META['QUERY_STRING']}")
-    except Exception:
-        pass
-
-    raise HttpError(404, "Object with given ID not found")
-
-
-### Tag Editor API Endpoints #########################################################################
-
-
-class TagAutocompleteSchema(Schema):
-    tags: list[dict]
-
-
-class TagCreateSchema(Schema):
-    name: str
-
-
-class TagCreateResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    tag_name: str
-    created: bool
-
-
-class TagSearchSnapshotSchema(Schema):
-    id: str
-    title: str
-    url: str
-    favicon_url: str
-    admin_url: str
-    archive_url: str
-    downloaded_at: str | None = None
-
-
-class TagSearchCardSchema(Schema):
-    id: int
-    name: str
-    slug: str
-    num_snapshots: int
-    filter_url: str
-    edit_url: str
-    export_urls_url: str
-    export_jsonl_url: str
-    rename_url: str
-    delete_url: str
-    snapshots: list[TagSearchSnapshotSchema]
-
-
-class TagSearchResponseSchema(Schema):
-    tags: list[TagSearchCardSchema]
-    sort: str
-    created_by: str
-    year: str
-    has_snapshots: str
-
-
-class TagUpdateSchema(Schema):
-    name: str
-
-
-class TagUpdateResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    tag_name: str
-    slug: str
-
-
-class TagDeleteResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    deleted_count: int
-
-
-class TagSnapshotRequestSchema(Schema):
-    snapshot_id: str
-    tag_name: str | None = None
-    tag_id: int | None = None
-
-
-class TagSnapshotResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    tag_name: str
-
-
-@router.get("/tags/search/", response=TagSearchResponseSchema, url_name="search_tags")
-def search_tags(
-    request: HttpRequest,
-    q: str = "",
-    sort: str = "created_desc",
-    created_by: str = "",
-    year: str = "",
-    has_snapshots: str = "all",
-):
-    """Return detailed tag cards for admin/live-search UIs."""
-    normalized_sort = normalize_tag_sort(sort)
-    normalized_created_by = normalize_created_by_filter(created_by)
-    normalized_year = normalize_created_year_filter(year)
-    normalized_has_snapshots = normalize_has_snapshots_filter(has_snapshots)
-    return {
-        "tags": build_tag_cards(
-            query=q,
-            request=request,
-            sort=normalized_sort,
-            created_by=normalized_created_by,
-            year=normalized_year,
-            has_snapshots=normalized_has_snapshots,
-        ),
-        "sort": normalized_sort,
-        "created_by": normalized_created_by,
-        "year": normalized_year,
-        "has_snapshots": normalized_has_snapshots,
-    }
-
-
-def _public_tag_listing_enabled() -> bool:
-    explicit = getattr(settings, "PUBLIC_SNAPSHOTS_LIST", None)
-    if explicit is not None:
-        return bool(explicit)
-    return bool(getattr(settings, "PUBLIC_INDEX", SERVER_CONFIG.PUBLIC_INDEX))
-
-
-def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    user = getattr(request, "user", None)
-    if getattr(user, "is_authenticated", False):
-        return True
-
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-
-    if token and auth_using_token(token=token, request=request):
-        return True
-
-    return _public_tag_listing_enabled()
-
-
-@router.get("/tags/autocomplete/", response=TagAutocompleteSchema, url_name="tags_autocomplete", auth=None)
-def tags_autocomplete(request: HttpRequest, q: str = ""):
-    """Return tags matching the query for autocomplete."""
-    if not _request_has_tag_autocomplete_access(request):
-        raise HttpError(401, "Authentication required")
-
-    tags = get_matching_tags(q)[: 50 if not q else 20]
-
-    return {
-        "tags": [{"id": tag.pk, "name": tag.name, "slug": tag.slug, "num_snapshots": getattr(tag, "num_snapshots", 0)} for tag in tags],
-    }
-
-
-@router.post("/tags/create/", response=TagCreateResponseSchema, url_name="tags_create")
-def tags_create(request: HttpRequest, data: TagCreateSchema):
-    """Create a new tag or return existing one."""
-    try:
-        tag, created = get_or_create_tag(
-            data.name,
-            created_by=request.user if request.user.is_authenticated else None,
-        )
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-        "created": created,
-    }
-
-
-@router.post("/tag/{tag_id}/rename", response=TagUpdateResponseSchema, url_name="rename_tag")
-def rename_tag(request: HttpRequest, tag_id: int, data: TagUpdateSchema):
-    try:
-        tag = rename_tag_record(get_tag_by_ref(tag_id), data.name)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-        "slug": tag.slug,
-    }
-
-
-@router.delete("/tag/{tag_id}", response=TagDeleteResponseSchema, url_name="delete_tag")
-def delete_tag(request: HttpRequest, tag_id: int):
-    try:
-        tag = get_tag_by_ref(tag_id)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-
-    deleted_count, _ = delete_tag_record(tag)
-    return {
-        "success": True,
-        "tag_id": int(tag_id),
-        "deleted_count": deleted_count,
-    }
-
-
-@router.get("/tag/{tag_id}/urls.txt", url_name="tag_urls_export")
-def tag_urls_export(request: HttpRequest, tag_id: int):
-    try:
-        tag = get_tag_by_ref(tag_id)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-
-    response = HttpResponse(export_tag_urls(tag), content_type="text/plain; charset=utf-8")
-    response["Content-Disposition"] = f'attachment; filename="tag-{tag.slug}-urls.txt"'
-    return response
-
-
-@router.get("/tag/{tag_id}/snapshots.jsonl", url_name="tag_snapshots_export")
-def tag_snapshots_export(request: HttpRequest, tag_id: int):
-    try:
-        tag = get_tag_by_ref(tag_id)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-
-    response = HttpResponse(export_tag_snapshots_jsonl(tag), content_type="application/x-ndjson; charset=utf-8")
-    response["Content-Disposition"] = f'attachment; filename="tag-{tag.slug}-snapshots.jsonl"'
-    return response
-
-
-@router.post("/tags/add-to-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_add_to_snapshot")
-def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
-    """Add a tag to a snapshot. Creates the tag if it doesn't exist."""
-    # Get the snapshot
-    try:
-        snapshot = Snapshot.objects.get(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        )
-    except Snapshot.DoesNotExist:
-        raise HttpError(404, "Snapshot not found")
-    except Snapshot.MultipleObjectsReturned:
-        snapshot = Snapshot.objects.filter(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        ).first()
-        if snapshot is None:
-            raise HttpError(404, "Snapshot not found")
-
-    # Get or create the tag
-    if data.tag_name:
-        try:
-            tag, _ = get_or_create_tag(
-                data.tag_name,
-                created_by=request.user if request.user.is_authenticated else None,
-            )
-        except ValueError as err:
-            raise HttpError(400, str(err)) from err
-    elif data.tag_id:
-        try:
-            tag = get_tag_by_ref(data.tag_id)
-        except Tag.DoesNotExist:
-            raise HttpError(404, "Tag not found")
-    else:
-        raise HttpError(400, "Either tag_name or tag_id is required")
-
-    # Add the tag to the snapshot
-    snapshot.tags.add(tag.pk)
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-    }
-
-
-@router.post("/tags/remove-from-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_remove_from_snapshot")
-def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
-    """Remove a tag from a snapshot."""
-    # Get the snapshot
-    try:
-        snapshot = Snapshot.objects.get(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        )
-    except Snapshot.DoesNotExist:
-        raise HttpError(404, "Snapshot not found")
-    except Snapshot.MultipleObjectsReturned:
-        snapshot = Snapshot.objects.filter(
-            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id),
-        ).first()
-        if snapshot is None:
-            raise HttpError(404, "Snapshot not found")
-
-    # Get the tag
-    if data.tag_id:
-        try:
-            tag = Tag.objects.get(pk=data.tag_id)
-        except Tag.DoesNotExist:
-            raise HttpError(404, "Tag not found")
-    elif data.tag_name:
-        try:
-            tag = Tag.objects.get(name__iexact=data.tag_name.strip())
-        except Tag.DoesNotExist:
-            raise HttpError(404, "Tag not found")
-    else:
-        raise HttpError(400, "Either tag_name or tag_id is required")
-
-    # Remove the tag from the snapshot
-    snapshot.tags.remove(tag.pk)
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-    }
@@ -1,201 +0,0 @@
-__package__ = "archivebox.api"
-
-from uuid import UUID
-from datetime import datetime
-from django.http import HttpRequest
-from django.utils import timezone
-
-from django.contrib.auth import get_user_model
-from django.contrib.auth.models import User
-
-from ninja import Router, Schema
-from ninja.errors import HttpError
-
-from archivebox.core.models import Snapshot
-from archivebox.crawls.models import Crawl
-
-from .auth import API_AUTH_METHODS
-
-router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
-
-
-class CrawlSchema(Schema):
-    TYPE: str = "crawls.models.Crawl"
-
-    id: UUID
-
-    modified_at: datetime
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-
-    status: str
-    retry_at: datetime | None
-
-    urls: str
-    max_depth: int
-    max_urls: int
-    max_size: int
-    tags_str: str
-    config: dict
-
-    # snapshots: List[SnapshotSchema]
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-
-    @staticmethod
-    def resolve_created_by_username(obj):
-        user_model = get_user_model()
-        user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
-        return username if isinstance(username, str) else str(user)
-
-    @staticmethod
-    def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
-            return obj.snapshot_set.all().distinct()
-        return Snapshot.objects.none()
-
-
-class CrawlUpdateSchema(Schema):
-    status: str | None = None
-    retry_at: datetime | None = None
-    tags: list[str] | None = None
-    tags_str: str | None = None
-
-
-class CrawlCreateSchema(Schema):
-    urls: list[str]
-    max_depth: int = 0
-    max_urls: int = 0
-    max_size: int = 0
-    tags: list[str] | None = None
-    tags_str: str = ""
-    label: str = ""
-    notes: str = ""
-    config: dict = {}
-
-
-class CrawlDeleteResponseSchema(Schema):
-    success: bool
-    crawl_id: str
-    deleted_count: int
-    deleted_snapshots: int
-
-
-def normalize_tag_list(tags: list[str] | None = None, tags_str: str = "") -> list[str]:
-    if tags is not None:
-        return [tag.strip() for tag in tags if tag and tag.strip()]
-    return [tag.strip() for tag in tags_str.split(",") if tag.strip()]
-
-
-@router.get("/crawls", response=list[CrawlSchema], url_name="get_crawls")
-def get_crawls(request: HttpRequest):
-    return Crawl.objects.all().distinct()
-
-
-@router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
-def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
-    urls = [url.strip() for url in data.urls if url and url.strip()]
-    if not urls:
-        raise HttpError(400, "At least one URL is required")
-    if data.max_depth not in (0, 1, 2, 3, 4):
-        raise HttpError(400, "max_depth must be between 0 and 4")
-    if data.max_urls < 0:
-        raise HttpError(400, "max_urls must be >= 0")
-    if data.max_size < 0:
-        raise HttpError(400, "max_size must be >= 0")
-
-    tags = normalize_tag_list(data.tags, data.tags_str)
-    crawl = Crawl.objects.create(
-        urls="\n".join(urls),
-        max_depth=data.max_depth,
-        max_urls=data.max_urls,
-        max_size=data.max_size,
-        tags_str=",".join(tags),
-        label=data.label,
-        notes=data.notes,
-        config=data.config,
-        status=Crawl.StatusChoices.QUEUED,
-        retry_at=timezone.now(),
-        created_by=request.user if isinstance(request.user, User) else None,
-    )
-    crawl.create_snapshots_from_urls()
-    return crawl
-
-
-@router.get("/crawl/{crawl_id}", response=CrawlSchema | str, url_name="get_crawl")
-def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
-    """Get a specific Crawl by id."""
-    setattr(request, "with_snapshots", with_snapshots)
-    setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
-
-    if crawl and as_rss:
-        # return snapshots as XML rss feed
-        urls = [
-            {"url": snapshot.url, "title": snapshot.title, "bookmarked_at": snapshot.bookmarked_at, "tags": snapshot.tags_str}
-            for snapshot in crawl.snapshot_set.all()
-        ]
-        xml = '<rss version="2.0"><channel>'
-        for url in urls:
-            xml += f"<item><url>{url['url']}</url><title>{url['title']}</title><bookmarked_at>{url['bookmarked_at']}</bookmarked_at><tags>{url['tags']}</tags></item>"
-        xml += "</channel></rss>"
-        return xml
-
-    return crawl
-
-
-@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
-def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
-    """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
-    payload = data.dict(exclude_unset=True)
-    update_fields = ["modified_at"]
-
-    tags = payload.pop("tags", None)
-    tags_str = payload.pop("tags_str", None)
-    if tags is not None or tags_str is not None:
-        crawl.tags_str = ",".join(normalize_tag_list(tags, tags_str or ""))
-        update_fields.append("tags_str")
-
-    if "status" in payload:
-        if payload["status"] not in Crawl.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        crawl.status = payload["status"]
-        if crawl.status == Crawl.StatusChoices.SEALED and "retry_at" not in payload:
-            crawl.retry_at = None
-        update_fields.append("status")
-
-    if "retry_at" in payload:
-        crawl.retry_at = payload["retry_at"]
-        update_fields.append("retry_at")
-
-    crawl.save(update_fields=update_fields)
-
-    if payload.get("status") == Crawl.StatusChoices.SEALED:
-        Snapshot.objects.filter(
-            crawl=crawl,
-            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
-        ).update(
-            status=Snapshot.StatusChoices.SEALED,
-            retry_at=None,
-            modified_at=timezone.now(),
-        )
-    return crawl
-
-
-@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
-def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
-    crawl_id_str = str(crawl.id)
-    snapshot_count = crawl.snapshot_set.count()
-    deleted_count, _ = crawl.delete()
-    return {
-        "success": True,
-        "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
-        "deleted_snapshots": snapshot_count,
-    }
@@ -1,161 +0,0 @@
-__package__ = "archivebox.api"
-
-from uuid import UUID
-from typing import Annotated
-from datetime import datetime
-
-from django.http import HttpRequest
-
-from ninja import FilterLookup, FilterSchema, Query, Router, Schema
-from ninja.pagination import paginate
-
-from archivebox.api.v1_core import CustomPagination
-
-
-router = Router(tags=["Machine and Dependencies"])
-
-
-# ============================================================================
-# Machine Schemas
-# ============================================================================
-
-
-class MachineSchema(Schema):
-    """Schema for Machine model."""
-
-    TYPE: str = "machine.Machine"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    guid: str
-    hostname: str
-    hw_in_docker: bool
-    hw_in_vm: bool
-    hw_manufacturer: str
-    hw_product: str
-    hw_uuid: str
-    os_arch: str
-    os_family: str
-    os_platform: str
-    os_release: str
-    os_kernel: str
-    stats: dict
-    num_uses_succeeded: int
-    num_uses_failed: int
-
-
-class MachineFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup("id__startswith")] = None
-    hostname: Annotated[str | None, FilterLookup("hostname__icontains")] = None
-    os_platform: Annotated[str | None, FilterLookup("os_platform__icontains")] = None
-    os_arch: Annotated[str | None, FilterLookup("os_arch")] = None
-    hw_in_docker: Annotated[bool | None, FilterLookup("hw_in_docker")] = None
-    hw_in_vm: Annotated[bool | None, FilterLookup("hw_in_vm")] = None
-    bin_providers: Annotated[str | None, FilterLookup("bin_providers__icontains")] = None
-
-
-# ============================================================================
-# Binary Schemas
-# ============================================================================
-
-
-class BinarySchema(Schema):
-    """Schema for Binary model."""
-
-    TYPE: str = "machine.Binary"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    machine_id: UUID
-    machine_hostname: str
-    name: str
-    binproviders: str
-    binprovider: str
-    abspath: str
-    version: str
-    sha256: str
-    status: str
-    is_valid: bool
-    num_uses_succeeded: int
-    num_uses_failed: int
-
-    @staticmethod
-    def resolve_machine_hostname(obj) -> str:
-        return obj.machine.hostname
-
-    @staticmethod
-    def resolve_is_valid(obj) -> bool:
-        return obj.is_valid
-
-
-class BinaryFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup("id__startswith")] = None
-    name: Annotated[str | None, FilterLookup("name__icontains")] = None
-    binprovider: Annotated[str | None, FilterLookup("binprovider")] = None
-    status: Annotated[str | None, FilterLookup("status")] = None
-    machine_id: Annotated[str | None, FilterLookup("machine_id__startswith")] = None
-    version: Annotated[str | None, FilterLookup("version__icontains")] = None
-
-
-# ============================================================================
-# Machine Endpoints
-# ============================================================================
-
-
-@router.get("/machines", response=list[MachineSchema], url_name="get_machines")
-@paginate(CustomPagination)
-def get_machines(request: HttpRequest, filters: Query[MachineFilterSchema]):
-    """List all machines."""
-    from archivebox.machine.models import Machine
-
-    return filters.filter(Machine.objects.all()).distinct()
-
-
-@router.get("/machine/current", response=MachineSchema, url_name="get_current_machine")
-def get_current_machine(request: HttpRequest):
-    """Get the current machine."""
-    from archivebox.machine.models import Machine
-
-    return Machine.current()
-
-
-@router.get("/machine/{machine_id}", response=MachineSchema, url_name="get_machine")
-def get_machine(request: HttpRequest, machine_id: str):
-    """Get a specific machine by ID."""
-    from archivebox.machine.models import Machine
-    from django.db.models import Q
-
-    return Machine.objects.get(Q(id__startswith=machine_id) | Q(hostname__iexact=machine_id))
-
-
-# ============================================================================
-
-
-# ============================================================================
-# Binary Endpoints
-# ============================================================================
-
-
-@router.get("/binaries", response=list[BinarySchema], url_name="get_binaries")
-@paginate(CustomPagination)
-def get_binaries(request: HttpRequest, filters: Query[BinaryFilterSchema]):
-    """List all binaries."""
-    from archivebox.machine.models import Binary
-
-    return filters.filter(Binary.objects.all().select_related("machine")).distinct()
-
-
-@router.get("/binary/{binary_id}", response=BinarySchema, url_name="get_binary")
-def get_binary(request: HttpRequest, binary_id: str):
-    """Get a specific binary by ID."""
-    from archivebox.machine.models import Binary
-
-    return Binary.objects.select_related("machine").get(id__startswith=binary_id)
-
-
-@router.get("/binary/by-name/{name}", response=list[BinarySchema], url_name="get_binaries_by_name")
-def get_binaries_by_name(request: HttpRequest, name: str):
-    """Get all binaries with the given name."""
-    from archivebox.machine.models import Binary
-
-    return list(Binary.objects.filter(name__iexact=name).select_related("machine"))
@@ -1 +0,0 @@
-__package__ = "archivebox.base_models"
@@ -1,726 +0,0 @@
-"""Base admin classes for models using UUIDv7."""
-
-__package__ = "archivebox.base_models"
-
-import json
-from collections.abc import Mapping
-from typing import NotRequired, TypedDict
-
-from django import forms
-from django.contrib import admin
-from django.db import models
-from django.forms.renderers import BaseRenderer
-from django.http import HttpRequest, QueryDict
-from django.utils.safestring import SafeString, mark_safe
-from django_object_actions import DjangoObjectActions
-
-
-class ConfigOption(TypedDict):
-    plugin: str
-    type: str | list[str]
-    default: object
-    description: str
-    enum: NotRequired[list[object]]
-    pattern: NotRequired[str]
-    minimum: NotRequired[int | float]
-    maximum: NotRequired[int | float]
-
-
-class KeyValueWidget(forms.Widget):
-    """
-    A widget that renders JSON dict as editable key-value input fields
-    with + and - buttons to add/remove rows.
-    Includes autocomplete for available config keys from the plugin system.
-    """
-
-    template_name = ""  # We render manually
-
-    class Media:
-        css = {
-            "all": [],
-        }
-        js = []
-
-    def _get_config_options(self) -> dict[str, ConfigOption]:
-        """Get available config options from plugins."""
-        try:
-            from archivebox.hooks import discover_plugin_configs
-
-            plugin_configs = discover_plugin_configs()
-            options: dict[str, ConfigOption] = {}
-            for plugin_name, schema in plugin_configs.items():
-                for key, prop in schema.get("properties", {}).items():
-                    option: ConfigOption = {
-                        "plugin": plugin_name,
-                        "type": prop.get("type", "string"),
-                        "default": prop.get("default", ""),
-                        "description": prop.get("description", ""),
-                    }
-                    for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in prop:
-                            option[schema_key] = prop[schema_key]
-                    options[key] = option
-            return options
-        except Exception:
-            return {}
-
-    def _parse_value(self, value: object) -> dict[str, object]:
-        # Parse JSON value to dict
-        if value is None:
-            return {}
-        if isinstance(value, str):
-            try:
-                parsed = json.loads(value) if value else {}
-            except json.JSONDecodeError:
-                return {}
-            return parsed if isinstance(parsed, dict) else {}
-        if isinstance(value, Mapping):
-            return {str(key): item for key, item in value.items()}
-        return {}
-
-    def render(
-        self,
-        name: str,
-        value: object,
-        attrs: Mapping[str, str] | None = None,
-        renderer: BaseRenderer | None = None,
-    ) -> SafeString:
-        data = self._parse_value(value)
-
-        widget_id = attrs.get("id", name) if attrs else name
-        config_options = self._get_config_options()
-
-        # Build datalist options
-        datalist_options = "\n".join(
-            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
-            for key, opt in sorted(config_options.items())
-        )
-
-        # Build config metadata as JSON for JS
-        config_meta_json = json.dumps(config_options)
-
-        html = f'''
-        <div id="{widget_id}_container" class="key-value-editor" style="max-width: 700px;">
-            <datalist id="{widget_id}_keys">
-                {datalist_options}
-            </datalist>
-            <div id="{widget_id}_rows" class="key-value-rows">
-        '''
-
-        # Render existing key-value pairs
-        for key, val in data.items():
-            val_str = json.dumps(val) if not isinstance(val, str) else val
-            html += self._render_row(widget_id, key, val_str)
-
-        # Always add one empty row for new entries
-        html += self._render_row(widget_id, "", "")
-
-        html += f'''
-            </div>
-            <div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
-                <button type="button" onclick="addKeyValueRow_{widget_id}()"
-                        style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
-                    + Add Row
-                </button>
-            </div>
-            <input type="hidden" name="{name}" id="{widget_id}" value="">
-            <script>
-                (function() {{
-                    var configMeta_{widget_id} = {config_meta_json};
-                    var rowCounter_{widget_id} = 0;
-
-                    function stringifyValue_{widget_id}(value) {{
-                        return typeof value === 'string' ? value : JSON.stringify(value);
-                    }}
-
-                    function getTypes_{widget_id}(meta) {{
-                        if (!meta || meta.type === undefined || meta.type === null) {{
-                            return [];
-                        }}
-                        return Array.isArray(meta.type) ? meta.type : [meta.type];
-                    }}
-
-                    function getMetaForKey_{widget_id}(key) {{
-                        if (!key) {{
-                            return null;
-                        }}
-
-                        var explicitMeta = configMeta_{widget_id}[key];
-                        if (explicitMeta) {{
-                            return Object.assign({{ key: key }}, explicitMeta);
-                        }}
-
-                        if (key.endsWith('_BINARY')) {{
-                            return {{
-                                key: key,
-                                plugin: 'custom',
-                                type: 'string',
-                                default: '',
-                                description: 'Path to binary executable',
-                            }};
-                        }}
-
-                        if (isRegexConfigKey_{widget_id}(key)) {{
-                            return {{
-                                key: key,
-                                plugin: 'custom',
-                                type: 'string',
-                                default: '',
-                                description: 'Regex pattern list',
-                            }};
-                        }}
-
-                        return null;
-                    }}
-
-                    function describeMeta_{widget_id}(meta) {{
-                        if (!meta) {{
-                            return '';
-                        }}
-
-                        var details = '';
-                        if (Array.isArray(meta.enum) && meta.enum.length) {{
-                            details = 'Allowed: ' + meta.enum.map(stringifyValue_{widget_id}).join(', ');
-                        }} else {{
-                            var types = getTypes_{widget_id}(meta);
-                            if (types.length) {{
-                                details = 'Expected: ' + types.join(' or ');
-                            }}
-                        }}
-
-                        if (meta.minimum !== undefined || meta.maximum !== undefined) {{
-                            var bounds = [];
-                            if (meta.minimum !== undefined) bounds.push('min ' + meta.minimum);
-                            if (meta.maximum !== undefined) bounds.push('max ' + meta.maximum);
-                            details += (details ? ' ' : '') + '(' + bounds.join(', ') + ')';
-                        }}
-
-                        return [meta.description || '', details].filter(Boolean).join(' ');
-                    }}
-
-                    function getExampleInput_{widget_id}(key, meta) {{
-                        var types = getTypes_{widget_id}(meta);
-                        if (key.endsWith('_BINARY')) {{
-                            return 'Example: wget or /usr/bin/wget';
-                        }}
-                        if (key.endsWith('_ARGS_EXTRA') || key.endsWith('_ARGS')) {{
-                            return 'Example: ["--extra-arg"]';
-                        }}
-                        if (types.includes('array')) {{
-                            return 'Example: ["value"]';
-                        }}
-                        if (types.includes('object')) {{
-                            if (key === 'SAVE_ALLOWLIST' || key === 'SAVE_DENYLIST') {{
-                                return 'Example: {{"^https://example\\\\.com": ["wget"]}}';
-                            }}
-                            return 'Example: {{"key": "value"}}';
-                        }}
-                        return '';
-                    }}
-
-                    function isRegexConfigKey_{widget_id}(key) {{
-                        return key === 'URL_ALLOWLIST' ||
-                            key === 'URL_DENYLIST' ||
-                            key === 'SAVE_ALLOWLIST' ||
-                            key === 'SAVE_DENYLIST' ||
-                            key.endsWith('_PATTERN') ||
-                            key.includes('REGEX');
-                    }}
-
-                    function isSimpleFilterPattern_{widget_id}(pattern) {{
-                        return /^[\\w.*:-]+$/.test(pattern);
-                    }}
-
-                    function validateRegexPattern_{widget_id}(pattern) {{
-                        if (!pattern || isSimpleFilterPattern_{widget_id}(pattern)) {{
-                            return '';
-                        }}
-
-                        try {{
-                            new RegExp(pattern);
-                        }} catch (error) {{
-                            return error && error.message ? error.message : 'Invalid regex';
-                        }}
-                        return '';
-                    }}
-
-                    function validateRegexConfig_{widget_id}(key, raw, typeName) {{
-                        if (typeName === 'object') {{
-                            var parsed;
-                            try {{
-                                parsed = JSON.parse(raw);
-                            }} catch (error) {{
-                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
-                            }}
-                            if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {{
-                                return {{ ok: false, value: parsed, message: 'Must be a JSON object' }};
-                            }}
-                            for (var regexKey in parsed) {{
-                                var objectRegexError = validateRegexPattern_{widget_id}(regexKey);
-                                if (objectRegexError) {{
-                                    return {{ ok: false, value: parsed, message: 'Invalid regex key "' + regexKey + '": ' + objectRegexError }};
-                                }}
-                            }}
-                            return {{ ok: true, value: parsed, message: '' }};
-                        }}
-
-                        var patterns = raw.split(/[\\n,]+/).map(function(pattern) {{
-                            return pattern.trim();
-                        }}).filter(Boolean);
-                        for (var i = 0; i < patterns.length; i++) {{
-                            var regexError = validateRegexPattern_{widget_id}(patterns[i]);
-                            if (regexError) {{
-                                return {{ ok: false, value: raw, message: 'Invalid regex "' + patterns[i] + '": ' + regexError }};
-                            }}
-                        }}
-                        return {{ ok: true, value: raw, message: '' }};
-                    }}
-
-                    function validateBinaryValue_{widget_id}(raw) {{
-                        if (!raw) {{
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        if (/['"`]/.test(raw)) {{
-                            return {{ ok: false, value: raw, message: 'Binary paths cannot contain quotes' }};
-                        }}
-
-                        if (/[;&|<>$(){{}}\\[\\]!]/.test(raw)) {{
-                            return {{ ok: false, value: raw, message: 'Binary paths can only be a binary name or absolute path' }};
-                        }}
-
-                        if (raw.startsWith('/')) {{
-                            if (/^[A-Za-z0-9_./+\\- ]+$/.test(raw)) {{
-                                return {{ ok: true, value: raw, message: '' }};
-                            }}
-                            return {{ ok: false, value: raw, message: 'Absolute paths may only contain path-safe characters' }};
-                        }}
-
-                        if (/^[A-Za-z0-9_.+-]+$/.test(raw)) {{
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        return {{ ok: false, value: raw, message: 'Enter a binary name like wget or an absolute path like /usr/bin/wget' }};
-                    }}
-
-                    function parseValue_{widget_id}(raw) {{
-                        try {{
-                            if (raw === 'true') return true;
-                            if (raw === 'false') return false;
-                            if (raw === 'null') return null;
-                            if (raw !== '' && !isNaN(raw)) return Number(raw);
-                            if ((raw.startsWith('{{') && raw.endsWith('}}')) ||
-                                (raw.startsWith('[') && raw.endsWith(']')) ||
-                                (raw.startsWith('"') && raw.endsWith('"'))) {{
-                                return JSON.parse(raw);
-                            }}
-                        }} catch (error) {{
-                            return raw;
-                        }}
-                        return raw;
-                    }}
-
-                    function sameValue_{widget_id}(left, right) {{
-                        return left === right || JSON.stringify(left) === JSON.stringify(right);
-                    }}
-
-                    function parseTypedValue_{widget_id}(raw, typeName, meta) {{
-                        var numberValue;
-                        var parsed;
-
-                        if (typeName && meta && meta.key && isRegexConfigKey_{widget_id}(meta.key)) {{
-                            return validateRegexConfig_{widget_id}(meta.key, raw, typeName);
-                        }}
-
-                        if (typeName === 'string' && meta && meta.key && meta.key.endsWith('_BINARY')) {{
-                            return validateBinaryValue_{widget_id}(raw);
-                        }}
-
-                        if (typeName === 'string') {{
-                            if (meta.pattern) {{
-                                try {{
-                                    if (!(new RegExp(meta.pattern)).test(raw)) {{
-                                        return {{ ok: false, value: raw, message: 'Must match pattern ' + meta.pattern }};
-                                    }}
-                                }} catch (error) {{}}
-                            }}
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        if (typeName === 'integer') {{
-                            if (!/^-?\\d+$/.test(raw)) {{
-                                return {{ ok: false, value: raw, message: 'Must be an integer' }};
-                            }}
-                            numberValue = Number(raw);
-                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
-                            }}
-                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
-                            }}
-                            return {{ ok: true, value: numberValue, message: '' }};
-                        }}
-
-                        if (typeName === 'number') {{
-                            if (raw === '' || isNaN(raw)) {{
-                                return {{ ok: false, value: raw, message: 'Must be a number' }};
-                            }}
-                            numberValue = Number(raw);
-                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
-                            }}
-                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
-                            }}
-                            return {{ ok: true, value: numberValue, message: '' }};
-                        }}
-
-                        if (typeName === 'boolean') {{
-                            var lowered = raw.toLowerCase();
-                            if (lowered === 'true' || raw === '1') return {{ ok: true, value: true, message: '' }};
-                            if (lowered === 'false' || raw === '0') return {{ ok: true, value: false, message: '' }};
-                            return {{ ok: false, value: raw, message: 'Must be true or false' }};
-                        }}
-
-                        if (typeName === 'null') {{
-                            return raw === 'null'
-                                ? {{ ok: true, value: null, message: '' }}
-                                : {{ ok: false, value: raw, message: 'Must be null' }};
-                        }}
-
-                        if (typeName === 'array' || typeName === 'object') {{
-                            try {{
-                                parsed = JSON.parse(raw);
-                            }} catch (error) {{
-                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
-                            }}
-
-                            if (typeName === 'array' && Array.isArray(parsed)) {{
-                                return {{ ok: true, value: parsed, message: '' }};
-                            }}
-                            if (typeName === 'object' && parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {{
-                                return {{ ok: true, value: parsed, message: '' }};
-                            }}
-
-                            return {{
-                                ok: false,
-                                value: parsed,
-                                message: typeName === 'array' ? 'Must be a JSON array' : 'Must be a JSON object',
-                            }};
-                        }}
-
-                        return {{ ok: true, value: parseValue_{widget_id}(raw), message: '' }};
-                    }}
-
-                    function validateValueAgainstMeta_{widget_id}(raw, meta) {{
-                        if (!meta || raw === '') {{
-                            return {{ state: 'neutral', value: raw, message: '' }};
-                        }}
-
-                        var enumValues = Array.isArray(meta.enum) ? meta.enum : [];
-                        var types = getTypes_{widget_id}(meta);
-                        if (!types.length) {{
-                            types = ['string'];
-                        }}
-
-                        var error = 'Invalid value';
-                        for (var i = 0; i < types.length; i++) {{
-                            var candidate = parseTypedValue_{widget_id}(raw, types[i], meta);
-                            if (!candidate.ok) {{
-                                error = candidate.message || error;
-                                continue;
-                            }}
-                            if (enumValues.length && !enumValues.some(function(enumValue) {{
-                                return sameValue_{widget_id}(enumValue, candidate.value) || stringifyValue_{widget_id}(enumValue) === raw;
-                            }})) {{
-                                error = 'Must be one of: ' + enumValues.map(stringifyValue_{widget_id}).join(', ');
-                                continue;
-                            }}
-                            return {{ state: 'valid', value: candidate.value, message: '' }};
-                        }}
-
-                        return {{ state: 'invalid', value: raw, message: error }};
-                    }}
-
-                    function ensureRowId_{widget_id}(row) {{
-                        if (!row.dataset.rowId) {{
-                            row.dataset.rowId = String(rowCounter_{widget_id}++);
-                        }}
-                        return row.dataset.rowId;
-                    }}
-
-                    function setRowHelp_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var help = row.querySelector('.kv-help');
-                        if (!keyInput || !help) {{
-                            return;
-                        }}
-
-                        var key = keyInput.value.trim();
-                        if (!key) {{
-                            help.textContent = '';
-                            return;
-                        }}
-
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (meta) {{
-                            var extra = isRegexConfigKey_{widget_id}(key)
-                                ? ((meta.type === 'object' || (Array.isArray(meta.type) && meta.type.includes('object')))
-                                    ? ' Expected: JSON object with regex keys.'
-                                    : ' Expected: valid regex.')
-                                : '';
-                            var example = getExampleInput_{widget_id}(key, meta);
-                            help.textContent = [describeMeta_{widget_id}(meta) + extra, example].filter(Boolean).join(' ');
-                        }} else {{
-                            help.textContent = 'Custom key';
-                        }}
-                    }}
-
-                    function configureValueInput_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var valueInput = row.querySelector('.kv-value');
-                        var datalist = row.querySelector('.kv-value-options');
-                        if (!keyInput || !valueInput || !datalist) {{
-                            return;
-                        }}
-
-                        var rowId = ensureRowId_{widget_id}(row);
-                        datalist.id = '{widget_id}_value_options_' + rowId;
-
-                        var meta = getMetaForKey_{widget_id}(keyInput.value.trim());
-                        var enumValues = Array.isArray(meta && meta.enum) ? meta.enum : [];
-                        var types = getTypes_{widget_id}(meta);
-                        if (!enumValues.length && types.includes('boolean')) {{
-                            enumValues = ['True', 'False'];
-                        }}
-                        if (enumValues.length) {{
-                            datalist.innerHTML = enumValues.map(function(enumValue) {{
-                                return '<option value="' + stringifyValue_{widget_id}(enumValue).replace(/"/g, '&quot;') + '"></option>';
-                            }}).join('');
-                            valueInput.setAttribute('list', datalist.id);
-                        }} else {{
-                            datalist.innerHTML = '';
-                            valueInput.removeAttribute('list');
-                        }}
-                    }}
-
-                    function setValueValidationState_{widget_id}(input, state, message) {{
-                        if (!input) {{
-                            return;
-                        }}
-
-                        if (state === 'valid') {{
-                            input.style.borderColor = '#2da44e';
-                            input.style.boxShadow = '0 0 0 1px rgba(45, 164, 78, 0.18)';
-                            input.style.backgroundColor = '#f6ffed';
-                        }} else if (state === 'invalid') {{
-                            input.style.borderColor = '#cf222e';
-                            input.style.boxShadow = '0 0 0 1px rgba(207, 34, 46, 0.18)';
-                            input.style.backgroundColor = '#fff8f8';
-                        }} else {{
-                            input.style.borderColor = '#ccc';
-                            input.style.boxShadow = 'none';
-                            input.style.backgroundColor = '';
-                        }}
-                        input.title = message || '';
-                    }}
-
-                    function applyValueValidation_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var valueInput = row.querySelector('.kv-value');
-                        if (!keyInput || !valueInput) {{
-                            return;
-                        }}
-
-                        var key = keyInput.value.trim();
-                        if (!key) {{
-                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
-                            return;
-                        }}
-
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (!meta) {{
-                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
-                            return;
-                        }}
-
-                        var validation = validateValueAgainstMeta_{widget_id}(valueInput.value.trim(), meta);
-                        setValueValidationState_{widget_id}(valueInput, validation.state, validation.message);
-                    }}
-
-                    function coerceValueForStorage_{widget_id}(key, raw) {{
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (!meta) {{
-                            return parseValue_{widget_id}(raw);
-                        }}
-
-                        var validation = validateValueAgainstMeta_{widget_id}(raw, meta);
-                        return validation.state === 'valid' ? validation.value : raw;
-                    }}
-
-                    function initializeRows_{widget_id}() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
-                            ensureRowId_{widget_id}(row);
-                            configureValueInput_{widget_id}(row);
-                            setRowHelp_{widget_id}(row);
-                            applyValueValidation_{widget_id}(row);
-                        }});
-                    }}
-
-                    function updateHiddenField_{widget_id}() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        var rows = container.querySelectorAll('.key-value-row');
-                        var result = {{}};
-                        rows.forEach(function(row) {{
-                            var keyInput = row.querySelector('.kv-key');
-                            var valInput = row.querySelector('.kv-value');
-                            if (keyInput && valInput && keyInput.value.trim()) {{
-                                var key = keyInput.value.trim();
-                                var val = valInput.value.trim();
-                                result[key] = coerceValueForStorage_{widget_id}(key, val);
-                            }}
-                        }});
-                        document.getElementById('{widget_id}').value = JSON.stringify(result);
-                    }}
-
-                    window.addKeyValueRow_{widget_id} = function() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        var newRow = document.createElement('div');
-                        newRow.className = 'key-value-row';
-                        newRow.style.cssText = 'margin-bottom: 6px;';
-                        newRow.innerHTML = '<div style="display: flex; gap: 8px; align-items: center;">' +
-                            '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
-                            'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
-                            '<input type="text" class="kv-value" placeholder="value" ' +
-                            'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
-                            '<datalist class="kv-value-options"></datalist>' +
-                            '<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
-                            'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>' +
-                            '</div>' +
-                            '<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>';
-                        container.appendChild(newRow);
-                        ensureRowId_{widget_id}(newRow);
-                        configureValueInput_{widget_id}(newRow);
-                        setRowHelp_{widget_id}(newRow);
-                        applyValueValidation_{widget_id}(newRow);
-                        updateHiddenField_{widget_id}();
-                        newRow.querySelector('.kv-key').focus();
-                    }};
-
-                    window.removeKeyValueRow_{widget_id} = function(btn) {{
-                        var row = btn.closest('.key-value-row');
-                        row.remove();
-                        updateHiddenField_{widget_id}();
-                    }};
-
-                    window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
-
-                    // Initialize on load
-                    document.addEventListener('DOMContentLoaded', function() {{
-                        initializeRows_{widget_id}();
-                        updateHiddenField_{widget_id}();
-                    }});
-                    // Also run immediately in case DOM is already ready
-                    if (document.readyState !== 'loading') {{
-                        initializeRows_{widget_id}();
-                        updateHiddenField_{widget_id}();
-                    }}
-
-                    // Update on any input change
-                    var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
-
-                    rowsEl_{widget_id}.addEventListener('input', function(event) {{
-                        var row = event.target.closest('.key-value-row');
-                        if (!row) {{
-                            return;
-                        }}
-
-                        if (event.target.classList.contains('kv-key')) {{
-                            configureValueInput_{widget_id}(row);
-                            setRowHelp_{widget_id}(row);
-                        }}
-
-                        if (event.target.classList.contains('kv-key') || event.target.classList.contains('kv-value')) {{
-                            applyValueValidation_{widget_id}(row);
-                            updateHiddenField_{widget_id}();
-                        }}
-                    }});
-                }})();
-            </script>
-        </div>
-        '''
-        return mark_safe(html)
-
-    def _render_row(self, widget_id: str, key: str, value: str) -> str:
-        return f'''
-            <div class="key-value-row" style="margin-bottom: 6px;">
-                <div style="display: flex; gap: 8px; align-items: center;">
-                    <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
-                           style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <input type="text" class="kv-value" value="{self._escape(value)}" placeholder="value"
-                           style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <datalist class="kv-value-options"></datalist>
-                    <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
-                            style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>
-                </div>
-                <div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>
-            </div>
-        '''
-
-    def _escape(self, s: object) -> str:
-        """Escape HTML special chars in attribute values."""
-        if not s:
-            return ""
-        return str(s).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
-
-    def value_from_datadict(
-        self,
-        data: QueryDict | Mapping[str, object],
-        files: object,
-        name: str,
-    ) -> str:
-        value = data.get(name, "{}")
-        return value if isinstance(value, str) else "{}"
-
-
-class ConfigEditorMixin(admin.ModelAdmin):
-    """
-    Mixin for admin classes with a config JSON field.
-
-    Provides a key-value editor widget with autocomplete for available config keys.
-    """
-
-    def formfield_for_dbfield(
-        self,
-        db_field: models.Field[object, object],
-        request: HttpRequest,
-        **kwargs: object,
-    ) -> forms.Field | None:
-        """Use KeyValueWidget for the config JSON field."""
-        if db_field.name == "config":
-            kwargs["widget"] = KeyValueWidget()
-        return super().formfield_for_dbfield(db_field, request, **kwargs)
-
-
-class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
-    list_display = ("id", "created_at", "created_by")
-    readonly_fields = ("id", "created_at", "modified_at")
-    show_search_mode_selector = False
-
-    def get_default_search_mode(self) -> str:
-        # The shared changelist template always asks every admin for a default
-        # search mode, even when the search-mode toggle is hidden.
-        return "meta"
-
-    def get_form(
-        self,
-        request: HttpRequest,
-        obj: models.Model | None = None,
-        change: bool = False,
-        **kwargs: object,
-    ):
-        form = super().get_form(request, obj, change=change, **kwargs)
-        if "created_by" in form.base_fields:
-            form.base_fields["created_by"].initial = request.user
-        return form
@@ -1,7 +0,0 @@
-# from django.apps import AppConfig
-
-
-# class BaseModelsConfig(AppConfig):
-#     default_auto_field = 'django.db.models.BigAutoField'
-
-#     name = 'base_models'
@@ -1,134 +0,0 @@
-"""Base models using UUIDv7 for all id fields."""
-
-__package__ = "archivebox.base_models"
-
-from archivebox.uuid_compat import uuid7
-from pathlib import Path
-
-from django.db import models
-from django.db.models import F
-from django.utils import timezone
-from django.contrib.auth import get_user_model
-from django.urls import reverse_lazy
-from django.conf import settings
-
-from django_stubs_ext.db.models import TypedModelMeta
-
-
-def get_or_create_system_user_pk(username="system"):
-    User = get_user_model()
-    # If there's exactly one superuser, use that for all system operations
-    if User.objects.filter(is_superuser=True).count() == 1:
-        return User.objects.filter(is_superuser=True).values_list("pk", flat=True)[0]
-    # Otherwise get or create the system user
-    user, _ = User.objects.get_or_create(
-        username=username,
-        defaults={"is_staff": True, "is_superuser": True, "email": "", "password": "!"},
-    )
-    return user.pk
-
-
-class AutoDateTimeField(models.DateTimeField):
-    """DateTimeField that automatically updates on save (legacy compatibility)."""
-
-    def pre_save(self, model_instance, add):
-        if add or not getattr(model_instance, self.attname):
-            value = timezone.now()
-            setattr(model_instance, self.attname, value)
-            return value
-        return super().pre_save(model_instance, add)
-
-
-class ModelWithUUID(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-    created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL,
-        on_delete=models.CASCADE,
-        default=get_or_create_system_user_pk,
-        null=False,
-        db_index=True,
-    )
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def __str__(self) -> str:
-        return f"[{self.id}] {self.__class__.__name__}"
-
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
-    @property
-    def api_url(self) -> str:
-        return str(reverse_lazy("api-1:get_any", args=[self.id]))
-
-    @property
-    def api_docs_url(self) -> str:
-        return f"/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}"
-
-
-class ModelWithNotes(models.Model):
-    """Mixin for models with a notes field."""
-
-    notes = models.TextField(blank=True, null=False, default="")
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-
-class ModelWithHealthStats(models.Model):
-    """Mixin for models with health tracking fields."""
-
-    num_uses_failed = models.PositiveIntegerField(default=0)
-    num_uses_succeeded = models.PositiveIntegerField(default=0)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    @property
-    def health(self) -> int:
-        total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
-        return round((self.num_uses_succeeded / total) * 100)
-
-    def increment_health_stats(self, success: bool):
-        """Atomically increment success or failure counter using F() expression."""
-        field = "num_uses_succeeded" if success else "num_uses_failed"
-        type(self).objects.filter(pk=self.pk).update(**{field: F(field) + 1})
-
-
-class ModelWithConfig(models.Model):
-    """Mixin for models with a JSON config field."""
-
-    config = models.JSONField(default=dict, null=True, blank=True, editable=True)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-
-class ModelWithOutputDir(ModelWithUUID):
-    class Meta(ModelWithUUID.Meta):
-        abstract = True
-
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        Path(self.output_dir).mkdir(parents=True, exist_ok=True)
-        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
-
-    @property
-    def output_dir_parent(self) -> str:
-        return f"{self._meta.model_name}s"
-
-    @property
-    def output_dir_name(self) -> str:
-        return str(self.id)
-
-    @property
-    def output_dir_str(self) -> str:
-        return f"{self.output_dir_parent}/{self.output_dir_name}"
-
-    @property
-    def output_dir(self) -> Path:
-        raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
@@ -1,199 +1,155 @@
-__package__ = "archivebox.cli"
-__command__ = "archivebox"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox'
+
 import os
 import sys
+import argparse
+
+from typing import Optional, Dict, List, IO, Union
+from pathlib import Path
+
+from ..config import OUTPUT_DIR, check_data_folder, check_migrations
+
 from importlib import import_module

-import rich_click as click
-from rich import print
+CLI_DIR = Path(__file__).resolve().parent

-from archivebox.config.version import VERSION
+# these common commands will appear sorted before any others for ease-of-use
+meta_cmds = ('help', 'version')                               # dont require valid data folder at all
+main_cmds = ('init', 'config', 'setup')                       # dont require existing db present
+archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
+fake_db = ("oneshot",)                                        # use fake in-memory db
+
+display_first = (*meta_cmds, *main_cmds, *archive_cmds)
+
+# every imported command module must have these properties in order to be valid
+required_attrs = ('__package__', '__command__', 'main')
+
+# basic checks to make sure imported files are valid subcommands
+is_cli_module = lambda fname: fname.startswith('archivebox_') and fname.endswith('.py')
+is_valid_cli_module = lambda module, subcommand: (
+    all(hasattr(module, attr) for attr in required_attrs)
+    and module.__command__.split(' ')[-1] == subcommand
+)


-if "--debug" in sys.argv:
-    os.environ["DEBUG"] = "True"
-    sys.argv.remove("--debug")
+def list_subcommands() -> Dict[str, str]:
+    """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
+
+    COMMANDS = []
+    for filename in os.listdir(CLI_DIR):
+        if is_cli_module(filename):
+            subcommand = filename.replace('archivebox_', '').replace('.py', '')
+            module = import_module('.archivebox_{}'.format(subcommand), __package__)
+            assert is_valid_cli_module(module, subcommand)
+            COMMANDS.append((subcommand, module.main.__doc__))
+            globals()[subcommand] = module.main
+
+    display_order = lambda cmd: (
+        display_first.index(cmd[0])
+        if cmd[0] in display_first else
+        100 + len(cmd[0])
+    )
+
+    return dict(sorted(COMMANDS, key=display_order))


-class ArchiveBoxGroup(click.Group):
-    """lazy loading click group for archivebox commands"""
+def run_subcommand(subcommand: str,
+                   subcommand_args: List[str]=None,
+                   stdin: Optional[IO]=None,
+                   pwd: Union[Path, str, None]=None) -> None:
+    """Run a given ArchiveBox subcommand with the given list of args"""

-    meta_commands = {
-        "help": "archivebox.cli.archivebox_help.main",
-        "version": "archivebox.cli.archivebox_version.main",
-        "mcp": "archivebox.cli.archivebox_mcp.main",
-    }
-    setup_commands = {
-        "init": "archivebox.cli.archivebox_init.main",
-        "install": "archivebox.cli.archivebox_install.main",
-    }
-    # Model commands (CRUD operations via subcommands)
-    model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot.main",
-        "archiveresult": "archivebox.cli.archivebox_archiveresult.main",
-        "tag": "archivebox.cli.archivebox_tag.main",
-        "binary": "archivebox.cli.archivebox_binary.main",
-        "process": "archivebox.cli.archivebox_process.main",
-        "machine": "archivebox.cli.archivebox_machine.main",
-        "persona": "archivebox.cli.archivebox_persona.main",
-    }
-    archive_commands = {
-        # High-level commands
-        "add": "archivebox.cli.archivebox_add.main",
-        "extract": "archivebox.cli.archivebox_extract.main",
-        "list": "archivebox.cli.archivebox_list.main",
-        "remove": "archivebox.cli.archivebox_remove.main",
-        "run": "archivebox.cli.archivebox_run.main",
-        "update": "archivebox.cli.archivebox_update.main",
-        "status": "archivebox.cli.archivebox_status.main",
-        "search": "archivebox.cli.archivebox_search.main",
-        "config": "archivebox.cli.archivebox_config.main",
-        "schedule": "archivebox.cli.archivebox_schedule.main",
-        "server": "archivebox.cli.archivebox_server.main",
-        "shell": "archivebox.cli.archivebox_shell.main",
-        "manage": "archivebox.cli.archivebox_manage.main",
-        # Introspection commands
-        "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
-    }
-    legacy_model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
-    }
-    all_subcommands = {
-        **meta_commands,
-        **setup_commands,
-        **model_commands,
-        **archive_commands,
-    }
-    renamed_commands = {
-        "setup": "install",
-        "import": "add",
-        "archive": "add",
-    }
-    legacy_model_subcommands = {
-        "crawl": {"create", "list", "update", "delete"},
-        "snapshot": {"create", "list", "update", "delete"},
-    }
+    subcommand_args = subcommand_args or []

-    @classmethod
-    def get_canonical_name(cls, cmd_name):
-        return cls.renamed_commands.get(cmd_name, cmd_name)
+    if subcommand not in meta_cmds:
+        from ..config import setup_django

-    @classmethod
-    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
-        if cmd_name not in cls.legacy_model_commands:
-            return False
+        cmd_requires_db = subcommand in archive_cmds
+        init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args

-        try:
-            arg_idx = sys.argv.index(cmd_name)
-        except ValueError:
-            return False
+        if cmd_requires_db:
+            check_data_folder(pwd)

-        remaining_args = sys.argv[arg_idx + 1 :]
-        if not remaining_args:
-            return False
+        setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)

-        first_arg = remaining_args[0]
-        if first_arg in ("-h", "--help"):
-            return False
+        if cmd_requires_db:
+            check_migrations()

-        return first_arg not in cls.legacy_model_subcommands[cmd_name]
-
-    def get_command(self, ctx, cmd_name):
-        # handle renamed commands
-        if cmd_name in self.renamed_commands:
-            new_name = self.renamed_commands[cmd_name]
-            print(
-                f" [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`",
-                file=sys.stderr,
-            )
-            cmd_name = new_name
-            ctx.invoked_subcommand = cmd_name
-
-        if self._should_use_legacy_model_command(cmd_name):
-            return self._lazy_load(self.legacy_model_commands[cmd_name])
-
-        # handle lazy loading of commands
-        if cmd_name in self.all_subcommands:
-            return self._lazy_load(cmd_name)
-
-        # fall-back to using click's default command lookup
-        return super().get_command(ctx, cmd_name)
-
-    @classmethod
-    def _lazy_load(cls, cmd_name_or_path):
-        import_path = cls.all_subcommands.get(cmd_name_or_path)
-        if import_path is None:
-            import_path = cmd_name_or_path
-        modname, funcname = import_path.rsplit(".", 1)
-
-        # print(f'LAZY LOADING {import_path}')
-        mod = import_module(modname)
-        func = getattr(mod, funcname)
-
-        if not hasattr(func, "__doc__"):
-            raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
-
-        # if not isinstance(cmd, click.BaseCommand):
-        # raise ValueError(f'lazy loading of {import_path} failed - not a click command')
-
-        return func
+    module = import_module('.archivebox_{}'.format(subcommand), __package__)
+    module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore


-@click.group(cls=ArchiveBoxGroup, invoke_without_command=True)
-@click.option("--help", "-h", is_flag=True, help="Show help")
-@click.version_option(VERSION, "-v", "--version", package_name="archivebox", message="%(version)s")
-@click.pass_context
-def cli(ctx, help=False):
-    """ArchiveBox: The self-hosted internet archive"""
+SUBCOMMANDS = list_subcommands()

-    subcommand = ArchiveBoxGroup.get_canonical_name(ctx.invoked_subcommand)
-
-    # if --help is passed or no subcommand is given, show custom help message
-    if help or ctx.invoked_subcommand is None:
-        ctx.invoke(ctx.command.get_command(ctx, "help"))
-
-    # if the subcommand is in archive_commands or model_commands,
-    # then we need to set up the django environment and check that we're in a valid data folder
-    if subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands:
-        # print('SETUP DJANGO AND CHECK DATA FOLDER')
-        try:
-            if subcommand == "server":
-                run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes")
-                if run_in_debug:
-                    os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
-                    if "--reload" in sys.argv:
-                        os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
-                        from archivebox.config.common import STORAGE_CONFIG
-
-                        os.environ["ARCHIVEBOX_RUNSERVER_PIDFILE"] = str(STORAGE_CONFIG.TMP_DIR / "runserver.pid")
-
-            from archivebox.config.django import setup_django
-            from archivebox.misc.checks import check_data_folder
-
-            setup_django()
-            check_data_folder()
-        except Exception as e:
-            print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]", file=sys.stderr)
-            if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
-                raise
+class NotProvided:
+    pass


-def main(args=None, prog_name=None, stdin=None):
-    # show `docker run archivebox xyz` in help messages if running in docker
-    IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
-    IS_TTY = sys.stdin.isatty()
-    prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
+def main(args: Optional[List[str]]=NotProvided, stdin: Optional[IO]=NotProvided, pwd: Optional[str]=None) -> None:
+    args = sys.argv[1:] if args is NotProvided else args
+    stdin = sys.stdin if stdin is NotProvided else stdin

-    # stdin param allows passing input data from caller (used by __main__.py)
-    # currently not used by click-based CLI, but kept for backwards compatibility
+    subcommands = list_subcommands()
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description='ArchiveBox: The self-hosted internet archive',
+        add_help=False,
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--help', '-h',
+        action='store_true',
+        help=subcommands['help'],
+    )
+    group.add_argument(
+        '--version',
+        action='store_true',
+        help=subcommands['version'],
+    )
+    group.add_argument(
+        "subcommand",
+        type=str,
+        help= "The name of the subcommand to run",
+        nargs='?',
+        choices=subcommands.keys(),
+        default=None,
+    )
+    parser.add_argument(
+        "subcommand_args",
+        help="Arguments for the subcommand",
+        nargs=argparse.REMAINDER,
+    )
+    command = parser.parse_args(args or ())

-    try:
-        cli(args=args, prog_name=prog_name)
-    except KeyboardInterrupt:
-        print("\n\n[red][X] Got CTRL+C. Exiting...[/red]")
+    if command.version:
+        command.subcommand = 'version'
+    elif command.help or command.subcommand is None:
+        command.subcommand = 'help'
+
+    if command.subcommand not in ('help', 'version', 'status'):
+        from ..logging_util import log_cli_command
+
+        log_cli_command(
+            subcommand=command.subcommand,
+            subcommand_args=command.subcommand_args,
+            stdin=stdin,
+            pwd=pwd or OUTPUT_DIR
+        )
+
+    run_subcommand(
+        subcommand=command.subcommand,
+        subcommand_args=command.subcommand_args,
+        stdin=stdin,
+        pwd=pwd or OUTPUT_DIR,
+    )
+
+
+__all__ = (
+    'SUBCOMMANDS',
+    'list_subcommands',
+    'run_subcommand',
+    *SUBCOMMANDS.keys(),
+)


-if __name__ == "__main__":
-    main()
@@ -1,286 +1,140 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
-__command__ = "archivebox add"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox add'

 import sys
-from pathlib import Path
+import argparse

-from typing import TYPE_CHECKING
+from typing import List, Optional, IO

-import rich_click as click
-
-from django.utils import timezone
-from django.db.models import QuerySet
-
-from archivebox.misc.util import enforce_types, docstring
-from archivebox.misc.util import parse_filesize_to_bytes
-from archivebox import CONSTANTS
-from archivebox.config.common import ARCHIVING_CONFIG, SERVER_CONFIG
-from archivebox.config.permissions import USER, HOSTNAME
+from ..main import add
+from ..util import docstring
+from ..parsers import PARSERS
+from ..config import OUTPUT_DIR, ONLY_NEW
+from ..logging_util import SmartFormatter, accept_stdin, stderr


-if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
-    from archivebox.crawls.models import Crawl
+@docstring(add.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=add.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--tag', '-t',
+        type=str,
+        default='',
+        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
+    )
+    parser.add_argument(
+        '--update', #'-u',
+        action='store_true',
+        default=not ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
+        help="Also retry previously skipped/failed links when adding new links",
+    )
+    parser.add_argument(
+        '--update-all', #'-n',
+        action='store_true',
+        default=False, 
+        help="Also update ALL links in index when finished adding new links",
+    )
+    parser.add_argument(
+        '--index-only', #'-o',
+        action='store_true',
+        help="Add the links to the main index without archiving them",
+    )
+    parser.add_argument(
+        'urls',
+        nargs='*',
+        type=str,
+        default=None,
+        help=(
+            'URLs or paths to archive e.g.:\n'
+            '    https://getpocket.com/users/USERNAME/feed/all\n'
+            '    https://example.com/some/rss/feed.xml\n'
+            '    https://example.com\n'
+            '    ~/Downloads/firefox_bookmarks_export.html\n'
+            '    ~/Desktop/sites_list.csv\n'
+        )
+    )
+    parser.add_argument(
+        "--depth",
+        action="store",
+        default=0,
+        choices=[0, 1],
+        type=int,
+        help="Recursively archive all linked pages up to this many hops away"
+    )
+    parser.add_argument(
+        "--overwrite",
+        default=False,
+        action="store_true",
+        help="Re-archive URLs from scratch, overwriting any existing files"
+    )
+    parser.add_argument(
+        "--init", #'-i',
+        action='store_true',
+        help="Init/upgrade the curent data directory before adding",
+    )
+    parser.add_argument(
+        "--extract",
+        type=str,
+        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
+              This does not take precedence over the configuration",
+        default=""
+    )
+    parser.add_argument(
+        "--parser",
+        type=str,
+        help="Parser used to read inputted URLs.",
+        default="auto",
+        choices=["auto", *PARSERS.keys()],
+    )
+    command = parser.parse_args(args or ())
+    urls = command.urls

+    stdin_urls = ''
+    if not urls:
+        stdin_urls = accept_stdin(stdin)

-def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
-    from archivebox.misc.jsonl import read_args_or_stdin
-
-    urls: list[str] = []
-    for record in read_args_or_stdin(args):
-        url = record.get("url")
-        if isinstance(url, str) and url:
-            urls.append(url)
-
-        urls_field = record.get("urls")
-        if isinstance(urls_field, str):
-            for line in urls_field.splitlines():
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    urls.append(line)
-
-    return urls
-
-
-@enforce_types
-def add(
-    urls: str | list[str],
-    depth: int | str = 0,
-    max_urls: int = 0,
-    max_size: int | str = 0,
-    tag: str = "",
-    url_allowlist: str = "",
-    url_denylist: str = "",
-    parser: str = "auto",
-    plugins: str = "",
-    persona: str = "Default",
-    overwrite: bool = False,
-    update: bool | None = None,
-    index_only: bool = False,
-    bg: bool = False,
-    created_by_id: int | None = None,
-) -> tuple["Crawl", QuerySet["Snapshot"]]:
-    """Add a new URL or list of URLs to your archive.
-
-    The flow is:
-    1. Save URLs to sources file
-    2. Create Crawl with URLs and max_depth
-    3. Crawl runner creates Snapshots from Crawl URLs (depth=0)
-    4. Crawl runner runs parser extractors on root snapshots
-    5. Parser extractors output to urls.jsonl
-    6. URLs are added to Crawl.urls and child Snapshots are created
-    7. Repeat until max_depth is reached
-    """
-
-    from rich import print
-
-    depth = int(depth)
-    max_urls = int(max_urls or 0)
-    max_size = parse_filesize_to_bytes(max_size)
-
-    if depth not in (0, 1, 2, 3, 4):
-        raise ValueError("Depth must be 0-4")
-    if max_urls < 0:
-        raise ValueError("max_urls must be >= 0")
-    if max_size < 0:
-        raise ValueError("max_size must be >= 0")
-
-    # import models once django is set up
-    from archivebox.core.models import Snapshot
-    from archivebox.crawls.models import Crawl
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.personas.models import Persona
-    from archivebox.misc.logging_util import printable_filesize
-    from archivebox.misc.system import get_dir_size
-    from archivebox.config.configset import get_config
-    from archivebox.services.runner import run_crawl
-
-    created_by_id = created_by_id or get_or_create_system_user_pk()
-    started_at = timezone.now()
-    if update is None:
-        update = not ARCHIVING_CONFIG.ONLY_NEW
-
-    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
-    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
-    sources_file.parent.mkdir(parents=True, exist_ok=True)
-    sources_file.write_text(urls if isinstance(urls, str) else "\n".join(urls))
-
-    # 2. Create a new Crawl with inline URLs
-    cli_args = [*sys.argv]
-    if cli_args[0].lower().endswith("archivebox"):
-        cli_args[0] = "archivebox"
-    cmd_str = " ".join(cli_args)
-
-    timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
-
-    # Read URLs directly into crawl
-    urls_content = sources_file.read_text()
-    persona_name = (persona or "Default").strip() or "Default"
-    plugins = plugins or str(get_config().get("PLUGINS") or "")
-    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
-    persona_obj.ensure_dirs()
-
-    crawl = Crawl.objects.create(
-        urls=urls_content,
-        max_depth=depth,
-        max_urls=max_urls,
-        max_size=max_size,
-        tags_str=tag,
-        persona_id=persona_obj.id,
-        label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
-        created_by_id=created_by_id,
-        config={
-            "ONLY_NEW": not update,
-            "INDEX_ONLY": index_only,
-            "OVERWRITE": overwrite,
-            "PLUGINS": plugins,
-            "DEFAULT_PERSONA": persona_name,
-            "PARSER": parser,
-            **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
-            **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
-        },
+    if (stdin_urls and urls) or (not stdin and not urls):
+        stderr(
+            '[X] You must pass URLs/paths to add via stdin or CLI arguments.\n',
+            color='red',
+        )
+        raise SystemExit(2)
+    add(
+        urls=stdin_urls or urls,
+        depth=command.depth,
+        tag=command.tag,
+        update=command.update,
+        update_all=command.update_all,
+        index_only=command.index_only,
+        overwrite=command.overwrite,
+        init=command.init,
+        extractors=command.extract,
+        parser=command.parser,
+        out_dir=pwd or OUTPUT_DIR,
    )

-    print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
-    first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
-    print(f"    [dim]First URL: {first_url}[/dim]")

-    # 3. The CrawlMachine will create Snapshots from all URLs when started
-    #    Parser extractors run on snapshots and discover more URLs
-    #    Discovered URLs become child Snapshots (depth+1)
-
-    if index_only:
-        # Just create the crawl but don't start processing
-        print("[yellow]\\[*] Index-only mode - crawl created but not started[/yellow]")
-        # Create snapshots for all URLs in the crawl
-        for url in crawl.get_urls_list():
-            snapshot, _ = Snapshot.objects.update_or_create(
-                crawl=crawl,
-                url=url,
-                defaults={
-                    "status": Snapshot.INITIAL_STATE,
-                    "retry_at": timezone.now(),
-                    "timestamp": str(timezone.now().timestamp()),
-                    "depth": 0,
-                },
-            )
-            if tag:
-                snapshot.save_tags(tag.split(","))
-            snapshot.ensure_crawl_symlink()
-        return crawl, crawl.snapshot_set.all()
-
-    if bg:
-        crawl.create_snapshots_from_urls()
-
-    # 5. Start the crawl runner to process the queue
-    #    The runner will:
-    #    - Process Crawl -> create Snapshots from all URLs
-    #    - Process Snapshots -> run extractors
-    #    - Parser extractors discover new URLs -> create child Snapshots
-    #    - Repeat until max_depth reached
-
-    if bg:
-        # Background mode: just queue work and return (background runner via server will pick it up)
-        print(
-            "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
-        )
-    else:
-        # Foreground mode: run full crawl runner until all work is done
-        print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
-        run_crawl(str(crawl.id))
-
-        # Print summary for foreground runs
-        try:
-            crawl.refresh_from_db()
-            snapshots_count = crawl.snapshot_set.count()
-            try:
-                from django.db.models import Count, Sum
-
-                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("archiveresult__output_size"))
-                total_bytes = int(totals["total_bytes"] or 0) if totals["snapshot_count"] else 0
-            except Exception:
-                total_bytes, _, _ = get_dir_size(crawl.output_dir)
-            total_size = printable_filesize(total_bytes)
-            total_time = timezone.now() - started_at
-            total_seconds = int(total_time.total_seconds())
-            mins, secs = divmod(total_seconds, 60)
-            hours, mins = divmod(mins, 60)
-            if hours:
-                duration_str = f"{hours}h {mins}m {secs}s"
-            elif mins:
-                duration_str = f"{mins}m {secs}s"
-            else:
-                duration_str = f"{secs}s"
-
-            # Output dir relative to DATA_DIR
-            try:
-                rel_output = Path(crawl.output_dir).relative_to(CONSTANTS.DATA_DIR)
-                rel_output_str = f"./{rel_output}"
-            except Exception:
-                rel_output_str = str(crawl.output_dir)
-
-            bind_addr = SERVER_CONFIG.BIND_ADDR or "127.0.0.1:8000"
-            if bind_addr.startswith("http://") or bind_addr.startswith("https://"):
-                base_url = bind_addr
-            else:
-                base_url = f"http://{bind_addr}"
-            admin_url = f"{base_url}/admin/crawls/crawl/{crawl.id}/change/"
-
-            print("\n[bold]crawl output saved to:[/bold]")
-            print(f"  {rel_output_str}")
-            print(f"  {admin_url}")
-            print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
-            print(f"[bold]total size:[/bold] {total_size}")
-            print(f"[bold]total time:[/bold] {duration_str}")
-        except Exception:
-            # Summary is best-effort; avoid failing the command if something goes wrong
-            pass
-
-    # 6. Return the list of Snapshots in this crawl
-    snapshots = crawl.snapshot_set.all()
-    return crawl, snapshots
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)


-@click.command()
-@click.option(
-    "--depth",
-    "-d",
-    type=click.Choice([str(i) for i in range(5)]),
-    default="0",
-    help="Recursively archive linked pages up to N hops away",
-)
-@click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
-@click.option("--max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
-@click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
-@click.option("--url-allowlist", "--domain-allowlist", default="", help="Comma-separated URL/domain allowlist for this crawl")
-@click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
-@click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
-@click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
-@click.option("--persona", default="Default", help="Authentication profile to use when archiving")
-@click.option("--overwrite", "-F", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
-@click.option("--update", is_flag=True, default=None, help="Retry any previously skipped/failed URLs when re-adding them")
-@click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
-@click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
-@click.argument("urls", nargs=-1, type=click.Path())
-@docstring(add.__doc__)
-def main(**kwargs):
-    """Add a new URL or list of URLs to your archive"""
-
-    raw_urls = kwargs.pop("urls")
-    urls = _collect_input_urls(raw_urls)
-    if not urls:
-        raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
-    if int(kwargs.get("max_urls") or 0) < 0:
-        raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
-    try:
-        kwargs["max_size"] = parse_filesize_to_bytes(kwargs.get("max_size"))
-    except ValueError as err:
-        raise click.BadParameter(str(err), param_hint="--max-size") from err
-
-    add(urls=urls, **kwargs)
-
-
-if __name__ == "__main__":
-    main()
+# TODO: Implement these
+#
+# parser.add_argument(
+#     '--mirror', #'-m',
+#     action='store_true',
+#     help='Archive an entire site (finding all linked pages below it on the same domain)',
+# )
+# parser.add_argument(
+#     '--crawler', #'-r',
+#     choices=('depth_first', 'breadth_first'),
+#     help='Controls which crawler to use in order to find outlinks in a given page',
+#     default=None,
+# )
@@ -1,387 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox archiveresult <action> [args...] [--filters]
-
-Manage ArchiveResult records (plugin extraction results).
-
-Actions:
-    create  - Create ArchiveResults for Snapshots (queue extractions)
-    list    - List ArchiveResults as JSONL (with optional filters)
-    update  - Update ArchiveResults from stdin JSONL
-    delete  - Delete ArchiveResults from stdin JSONL
-
-Examples:
-    # Create ArchiveResults for snapshots (queue for extraction)
-    archivebox snapshot list --status=queued | archivebox archiveresult create
-    archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
-
-    # List with filters
-    archivebox archiveresult list --status=failed
-    archivebox archiveresult list --plugin=screenshot --status=succeeded
-
-    # Update (reset failed extractions to queued)
-    archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued
-
-    # Delete
-    archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
-
-    # Re-run failed extractions
-    archivebox archiveresult list --status=failed | archivebox run
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox archiveresult"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_utils import apply_filters
-
-
-def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
-    return {
-        "type": "ArchiveResult",
-        "snapshot_id": str(snapshot_id),
-        "plugin": plugin,
-        "hook_name": hook_name,
-        "status": status,
-    }
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_archiveresults(
-    snapshot_id: str | None = None,
-    plugin: str | None = None,
-    status: str = "queued",
-) -> int:
-    """
-    Create ArchiveResult request records for Snapshots.
-
-    Reads Snapshot records from stdin and emits ArchiveResult request JSONL.
-    Pass-through: Non-Snapshot/ArchiveResult records are output unchanged.
-    If --plugin is specified, only emits requests for that plugin.
-    Otherwise, emits requests for all enabled snapshot hooks.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.config.configset import get_config
-    from archivebox.hooks import discover_hooks
-    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
-    from archivebox.core.models import Snapshot
-
-    is_tty = sys.stdout.isatty()
-
-    # If snapshot_id provided directly, use that
-    if snapshot_id:
-        try:
-            snapshots = [Snapshot.objects.get(id=snapshot_id)]
-            pass_through_records = []
-        except Snapshot.DoesNotExist:
-            rprint(f"[red]Snapshot not found: {snapshot_id}[/red]", file=sys.stderr)
-            return 1
-    else:
-        # Read from stdin
-        records = list(read_stdin())
-        if not records:
-            rprint("[yellow]No Snapshot records provided via stdin[/yellow]", file=sys.stderr)
-            return 1
-
-        # Separate snapshot records from pass-through records
-        snapshot_ids = []
-        pass_through_records = []
-
-        for record in records:
-            record_type = record.get("type", "")
-
-            if record_type == TYPE_SNAPSHOT:
-                # Pass through the Snapshot record itself
-                pass_through_records.append(record)
-                if record.get("id"):
-                    snapshot_ids.append(record["id"])
-
-            elif record_type == TYPE_ARCHIVERESULT:
-                # ArchiveResult records: pass through if they have an id
-                if record.get("id"):
-                    pass_through_records.append(record)
-                # If no id, we could create it, but for now just pass through
-                else:
-                    pass_through_records.append(record)
-
-            elif record_type:
-                # Other typed records (Crawl, Tag, etc): pass through
-                pass_through_records.append(record)
-
-            elif record.get("id"):
-                # Untyped record with id - assume it's a snapshot ID
-                snapshot_ids.append(record["id"])
-
-        # Output pass-through records first
-        if not is_tty:
-            for record in pass_through_records:
-                write_record(record)
-
-        if not snapshot_ids:
-            if pass_through_records:
-                rprint(f"[dim]Passed through {len(pass_through_records)} records, no new snapshots to process[/dim]", file=sys.stderr)
-                return 0
-            rprint("[yellow]No valid Snapshot IDs in input[/yellow]", file=sys.stderr)
-            return 1
-
-        snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids))
-
-    if not snapshots:
-        rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
-        return 0 if pass_through_records else 1
-
-    created_count = 0
-    for snapshot in snapshots:
-        if plugin:
-            if not is_tty:
-                write_record(build_archiveresult_request(snapshot.id, plugin, status=status))
-            created_count += 1
-        else:
-            config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
-            hooks = discover_hooks("Snapshot", config=config)
-            for hook_path in hooks:
-                hook_name = hook_path.name
-                plugin_name = hook_path.parent.name
-                if not is_tty:
-                    write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
-                created_count += 1
-
-    rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_archiveresults(
-    status: str | None = None,
-    plugin: str | None = None,
-    snapshot_id: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List ArchiveResults as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import ArchiveResult
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = ArchiveResult.objects.all().order_by("-start_ts")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "plugin": plugin,
-        "snapshot_id": snapshot_id,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for result in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "succeeded": "green",
-                "failed": "red",
-                "skipped": "dim",
-                "noresults": "dim",
-                "backoff": "magenta",
-            }.get(result.status, "dim")
-            rprint(
-                f"[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
-            )
-        else:
-            write_record(result.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} archive results[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_archiveresults(
-    status: str | None = None,
-) -> int:
-    """
-    Update ArchiveResults from stdin JSONL.
-
-    Reads ArchiveResult records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.core.models import ArchiveResult
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        result_id = record.get("id")
-        if not result_id:
-            continue
-
-        try:
-            result = ArchiveResult.objects.get(id=result_id)
-
-            # Apply updates from CLI flags
-            if status:
-                result.status = status
-
-            result.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(result.to_json())
-
-        except ArchiveResult.DoesNotExist:
-            rprint(f"[yellow]ArchiveResult not found: {result_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} archive results[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete ArchiveResults from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.core.models import ArchiveResult
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    result_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not result_ids:
-        rprint("[yellow]No valid archive result IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    results = ArchiveResult.objects.filter(id__in=result_ids)
-    count = results.count()
-
-    if count == 0:
-        rprint("[yellow]No matching archive results found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} archive results (dry run)[/yellow]", file=sys.stderr)
-        for result in results[:10]:
-            rprint(f"  [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}", file=sys.stderr)
-        if count > 10:
-            rprint(f"  ... and {count - 10} more", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = results.delete()
-    rprint(f"[green]Deleted {deleted_count} archive results[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage ArchiveResult records (plugin extraction results)."""
-    pass
-
-
-@main.command("create")
-@click.option("--snapshot-id", help="Snapshot ID to create results for")
-@click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
-    """Create ArchiveResults for Snapshots from stdin JSONL."""
-    sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
-
-
-@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, succeeded, failed, skipped)")
-@click.option("--plugin", "-p", help="Filter by plugin name")
-@click.option("--snapshot-id", help="Filter by snapshot ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(
-    status: str | None,
-    plugin: str | None,
-    snapshot_id: str | None,
-    limit: int | None,
-):
-    """List ArchiveResults as JSONL."""
-    sys.exit(
-        list_archiveresults(
-            status=status,
-            plugin=plugin,
-            snapshot_id=snapshot_id,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--status", "-s", help="Set status")
-def update_cmd(status: str | None):
-    """Update ArchiveResults from stdin JSONL."""
-    sys.exit(update_archiveresults(status=status))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete ArchiveResults from stdin JSONL."""
-    sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,319 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox binary <action> [args...] [--filters]
-
-Manage Binary records (detected executables like chrome, wget, etc.).
-
-Actions:
-    create  - Create/register a Binary
-    list    - List Binaries as JSONL (with optional filters)
-    update  - Update Binaries from stdin JSONL
-    delete  - Delete Binaries from stdin JSONL
-
-Examples:
-    # List all binaries
-    archivebox binary list
-
-    # List specific binary
-    archivebox binary list --name=chrome
-
-    # List binaries with specific version
-    archivebox binary list --version__icontains=120
-
-    # Delete old binary entries
-    archivebox binary list --name=chrome | archivebox binary delete --yes
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox binary"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_utils import apply_filters
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_binary(
-    name: str,
-    abspath: str,
-    version: str = "",
-) -> int:
-    """
-    Create/register a Binary.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    if not name or not abspath:
-        rprint("[red]Both --name and --abspath are required[/red]", file=sys.stderr)
-        return 1
-
-    try:
-        from archivebox.machine.models import Machine
-
-        machine = Machine.current()
-        created = not Binary.objects.filter(
-            machine=machine,
-            name=name,
-            abspath=abspath,
-            version=version,
-        ).exists()
-
-        # Mirror the Binary model lifecycle used elsewhere in the system so CLI
-        # records are owned by the current machine and can be safely piped into
-        # `archivebox run` without creating invalid rows missing machine_id.
-        binary = Binary.from_json(
-            {
-                "name": name,
-                "abspath": abspath,
-                "version": version,
-                "binproviders": "env",
-                "binprovider": "env",
-            },
-        )
-        if binary is None:
-            raise ValueError("failed to create binary record")
-
-        if not is_tty:
-            write_record(binary.to_json())
-
-        if created:
-            rprint(f"[green]Created binary: {name} at {abspath}[/green]", file=sys.stderr)
-        else:
-            rprint(f"[dim]Binary already exists: {name} at {abspath}[/dim]", file=sys.stderr)
-
-        return 0
-
-    except Exception as e:
-        rprint(f"[red]Error creating binary: {e}[/red]", file=sys.stderr)
-        return 1
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_binaries(
-    name: str | None = None,
-    abspath__icontains: str | None = None,
-    version__icontains: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Binaries as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Binary.objects.all().order_by("name", "-modified_at", "-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "name": name,
-        "abspath__icontains": abspath__icontains,
-        "version__icontains": version__icontains,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for binary in queryset:
-        if is_tty:
-            rprint(f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}")
-        else:
-            write_record(binary.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} binaries[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_binaries(
-    version: str | None = None,
-    abspath: str | None = None,
-) -> int:
-    """
-    Update Binaries from stdin JSONL.
-
-    Reads Binary records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        binary_id = record.get("id")
-        if not binary_id:
-            continue
-
-        try:
-            binary = Binary.objects.get(id=binary_id)
-
-            # Apply updates from CLI flags
-            if version:
-                binary.version = version
-            if abspath:
-                binary.abspath = abspath
-
-            binary.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(binary.to_json())
-
-        except Binary.DoesNotExist:
-            rprint(f"[yellow]Binary not found: {binary_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} binaries[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_binaries(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Binaries from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.machine.models import Binary
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    binary_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not binary_ids:
-        rprint("[yellow]No valid binary IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    binaries = Binary.objects.filter(id__in=binary_ids)
-    count = binaries.count()
-
-    if count == 0:
-        rprint("[yellow]No matching binaries found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} binaries (dry run)[/yellow]", file=sys.stderr)
-        for binary in binaries:
-            rprint(f"  {binary.name} {binary.abspath}", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = binaries.delete()
-    rprint(f"[green]Deleted {deleted_count} binaries[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Binary records (detected executables)."""
-    pass
-
-
-@main.command("create")
-@click.option("--name", "-n", required=True, help="Binary name (e.g., chrome, wget)")
-@click.option("--abspath", "-p", required=True, help="Absolute path to binary")
-@click.option("--version", "-v", default="", help="Binary version")
-def create_cmd(name: str, abspath: str, version: str):
-    """Create/register a Binary."""
-    sys.exit(create_binary(name=name, abspath=abspath, version=version))
-
-
-@main.command("list")
-@click.option("--name", "-n", help="Filter by name")
-@click.option("--abspath__icontains", help="Filter by path contains")
-@click.option("--version__icontains", help="Filter by version contains")
-@click.option("--limit", type=int, help="Limit number of results")
-def list_cmd(
-    name: str | None,
-    abspath__icontains: str | None,
-    version__icontains: str | None,
-    limit: int | None,
-):
-    """List Binaries as JSONL."""
-    sys.exit(
-        list_binaries(
-            name=name,
-            abspath__icontains=abspath__icontains,
-            version__icontains=version__icontains,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--version", "-v", help="Set version")
-@click.option("--abspath", "-p", help="Set path")
-def update_cmd(version: str | None, abspath: str | None):
-    """Update Binaries from stdin JSONL."""
-    sys.exit(update_binaries(version=version, abspath=abspath))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Binaries from stdin JSONL."""
-    sys.exit(delete_binaries(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,177 +1,64 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox config'

 import sys
-import rich_click as click
-from rich import print
-from benedict import benedict
+import argparse

-from archivebox.misc.util import docstring, enforce_types
-from archivebox.misc.toml_util import CustomTOMLEncoder
+from typing import Optional, List, IO
+
+from ..main import config
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, accept_stdin


-@enforce_types
-def config(
-    *keys,
-    get: bool = False,
-    set: bool = False,
-    search: bool = False,
-    reset: bool = False,
-    **kwargs,
-) -> None:
-    """Get and set your ArchiveBox project configuration values"""
-
-    from archivebox.misc.checks import check_data_folder
-    from archivebox.misc.logging_util import printable_config
-    from archivebox.config.collection import load_all_config, write_config_file, get_real_name
-    from archivebox.config.configset import get_flat_config, get_all_configs
-
-    check_data_folder()
-
-    FLAT_CONFIG = get_flat_config()
-    CONFIGS = get_all_configs()
-
-    config_options: list[str] = list(kwargs.pop("key=value", []) or keys or [f"{key}={val}" for key, val in kwargs.items()])
-    no_args = not (get or set or reset or config_options)
-
-    matching_config = {}
-    if search:
-        if config_options:
-            config_options = [get_real_name(key) for key in config_options]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            for config_section in CONFIGS.values():
-                aliases = getattr(config_section, "aliases", {})
-
-                for search_key in config_options:
-                    # search all aliases in the section
-                    for alias_key, key in aliases.items():
-                        if search_key.lower() in alias_key.lower():
-                            matching_config[key] = dict(config_section)[key]
-
-                    # search all keys and values in the section
-                    for existing_key, value in dict(config_section).items():
-                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
-                            matching_config[existing_key] = value
-
-        print(printable_config(matching_config))
-        raise SystemExit(not matching_config)
-
-    elif get or no_args:
-        if config_options:
-            config_options = [get_real_name(key) for key in config_options]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
-            if failed_config:
-                print("\n[red][X] These options failed to get[/red]")
-                print("    {}".format("\n    ".join(config_options)))
-                raise SystemExit(1)
-        else:
-            matching_config = FLAT_CONFIG
-
-        # Display core config sections
-        for config_section in CONFIGS.values():
-            section_header = getattr(config_section, "toml_section_header", "")
-            if isinstance(section_header, str) and section_header:
-                print(f"[grey53]\\[{section_header}][/grey53]")
-            else:
-                print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
-
-            kv_in_section = {key: val for key, val in dict(config_section).items() if key in matching_config}
-            print(benedict(kv_in_section).to_toml(encoder=CustomTOMLEncoder()).strip().replace("\n\n", "\n"))
-            print("[grey53]################################################################[/grey53]")
-
-        # Display plugin config section
-        from archivebox.hooks import discover_plugin_configs
-
-        plugin_configs = discover_plugin_configs()
-        plugin_keys = {}
-
-        # Collect all plugin config keys
-        for plugin_name, schema in plugin_configs.items():
-            if "properties" not in schema:
-                continue
-            for key in schema["properties"].keys():
-                if key in matching_config:
-                    plugin_keys[key] = matching_config[key]
-
-        # Display all plugin config in single [PLUGINS] section
-        if plugin_keys:
-            print("[grey53]\\[PLUGINS][/grey53]")
-            print(benedict(plugin_keys).to_toml(encoder=CustomTOMLEncoder()).strip().replace("\n\n", "\n"))
-            print("[grey53]################################################################[/grey53]")
-
-        raise SystemExit(not matching_config)
-
-    elif set:
-        new_config = {}
-        failed_options = []
-        for line in config_options:
-            if line.startswith("#") or not line.strip():
-                continue
-            if "=" not in line:
-                print("[red][X] Config KEY=VALUE must have an = sign in it[/red]")
-                print(f"    {line}")
-                raise SystemExit(2)
-
-            raw_key, val = line.split("=", 1)
-            raw_key = raw_key.upper().strip()
-            key = get_real_name(raw_key)
-            if key != raw_key:
-                print(
-                    f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
-                )
-
-            if key in FLAT_CONFIG:
-                new_config[key] = val.strip()
-            else:
-                failed_options.append(line)
-
-        if new_config:
-            before = FLAT_CONFIG
-            matching_config = write_config_file(new_config)
-            after = {**load_all_config(), **get_flat_config()}
-            print(printable_config(matching_config))
-
-            side_effect_changes = {}
-            for key, val in after.items():
-                if key in FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
-                    side_effect_changes[key] = after[key]
-
-            if side_effect_changes:
-                print(file=sys.stderr)
-                print("[yellow][i] Note: This change also affected these other options that depended on it:[/yellow]", file=sys.stderr)
-                print("    {}".format(printable_config(side_effect_changes, prefix="    ")), file=sys.stderr)
-
-        if failed_options:
-            print()
-            print("[red][X] These options failed to set (check for typos):[/red]")
-            print("    {}".format("\n    ".join(failed_options)))
-            raise SystemExit(1)
-
-    elif reset:
-        print("[red][X] This command is not implemented yet.[/red]")
-        print("    Please manually remove the relevant lines from your config file:")
-        raise SystemExit(2)
-
-    else:
-        print("[red][X] You must pass either --get or --set, or no arguments to get the whole config.[/red]")
-        print("    archivebox config")
-        print("    archivebox config --get SOME_KEY")
-        print("    archivebox config --set SOME_KEY=SOME_VALUE")
-        raise SystemExit(2)
-
-
-@click.command()
-@click.option("--search", is_flag=True, help="Search config KEYs, VALUEs, and ALIASES for the given term")
-@click.option("--get", is_flag=True, help="Get the value for the given config KEYs")
-@click.option("--set", is_flag=True, help="Set the given KEY=VALUE config values")
-@click.option("--reset", is_flag=True, help="Reset the given KEY config values to their defaults")
-@click.argument("KEY=VALUE", nargs=-1, type=str)
@docstring(config.__doc__)
-def main(**kwargs) -> None:
-    config(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=config.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--get', #'-g',
+        action='store_true',
+        help="Get the value for the given config KEYs",
+    )
+    group.add_argument(
+        '--set', #'-s',
+        action='store_true',
+        help="Set the given KEY=VALUE config values",
+    )
+    group.add_argument(
+        '--reset', #'-s',
+        action='store_true',
+        help="Reset the given KEY config values to their defaults",
+    )
+    parser.add_argument(
+        'config_options',
+        nargs='*',
+        type=str,
+        help='KEY or KEY=VALUE formatted config values to get or set',
+    )
+    command = parser.parse_args(args or ())
+
+    config_options_str = ''
+    if not command.config_options:
+        config_options_str = accept_stdin(stdin)
+
+    config(
+        config_options_str=config_options_str,
+        config_options=command.config_options,
+        get=command.get,
+        set=command.set,
+        reset=command.reset,
+        out_dir=pwd or OUTPUT_DIR,
+    )


-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,384 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox crawl <action> [args...] [--filters]
-
-Manage Crawl records.
-
-Actions:
-    create  - Create Crawl jobs from URLs
-    list    - List Crawls as JSONL (with optional filters)
-    update  - Update Crawls from stdin JSONL
-    delete  - Delete Crawls from stdin JSONL
-
-Examples:
-    # Create
-    archivebox crawl create https://example.com https://foo.com --depth=1
-    archivebox crawl create --tag=news https://example.com
-
-    # List with filters
-    archivebox crawl list --status=queued
-    archivebox crawl list --urls__icontains=example.com
-
-    # Update
-    archivebox crawl list --status=started | archivebox crawl update --status=queued
-
-    # Delete
-    archivebox crawl list --urls__icontains=spam.com | archivebox crawl delete --yes
-
-    # Full pipeline
-    archivebox crawl create https://example.com | archivebox snapshot create | archivebox run
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-from collections.abc import Iterable
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_utils import apply_filters
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_crawl(
-    urls: Iterable[str],
-    depth: int = 0,
-    tag: str = "",
-    status: str = "queued",
-    created_by_id: int | None = None,
-) -> int:
-    """
-    Create a Crawl job from URLs.
-
-    Takes URLs as args or stdin, creates one Crawl with all URLs, outputs JSONL.
-    Pass-through: Records that are not URLs are output unchanged (for piping).
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.crawls.models import Crawl
-
-    created_by_id = created_by_id or get_or_create_system_user_pk()
-    is_tty = sys.stdout.isatty()
-
-    # Collect all input records
-    records = list(read_args_or_stdin(urls))
-
-    if not records:
-        rprint("[yellow]No URLs provided. Pass URLs as arguments or via stdin.[/yellow]", file=sys.stderr)
-        return 1
-
-    # Separate pass-through records from URL records
-    url_list = []
-    pass_through_records = []
-
-    for record in records:
-        record_type = record.get("type", "")
-
-        # Pass-through: output records that aren't URL/Crawl types
-        if record_type and record_type != TYPE_CRAWL and not record.get("url") and not record.get("urls"):
-            pass_through_records.append(record)
-            continue
-
-        # Handle existing Crawl records (just pass through with id)
-        if record_type == TYPE_CRAWL and record.get("id"):
-            pass_through_records.append(record)
-            continue
-
-        # Collect URLs
-        url = record.get("url")
-        if url:
-            url_list.append(url)
-
-        # Handle 'urls' field (newline-separated)
-        urls_field = record.get("urls")
-        if urls_field:
-            for line in urls_field.split("\n"):
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    url_list.append(line)
-
-    # Output pass-through records first
-    if not is_tty:
-        for record in pass_through_records:
-            write_record(record)
-
-    if not url_list:
-        if pass_through_records:
-            # If we had pass-through records but no URLs, that's OK
-            rprint(f"[dim]Passed through {len(pass_through_records)} records, no new URLs[/dim]", file=sys.stderr)
-            return 0
-        rprint("[red]No valid URLs found[/red]", file=sys.stderr)
-        return 1
-
-    try:
-        # Build crawl record with all URLs as newline-separated string
-        crawl_record = {
-            "urls": "\n".join(url_list),
-            "max_depth": depth,
-            "tags_str": tag,
-            "status": status,
-            "label": "",
-        }
-
-        crawl = Crawl.from_json(crawl_record, overrides={"created_by_id": created_by_id})
-        if not crawl:
-            rprint("[red]Failed to create crawl[/red]", file=sys.stderr)
-            return 1
-
-        # Output JSONL record (only when piped)
-        if not is_tty:
-            write_record(crawl.to_json())
-
-        rprint(f"[green]Created crawl with {len(url_list)} URLs[/green]", file=sys.stderr)
-
-        # If TTY, show human-readable output
-        if is_tty:
-            rprint(f"  [dim]{crawl.id}[/dim]", file=sys.stderr)
-            for url in url_list[:5]:  # Show first 5 URLs
-                rprint(f"    {url[:70]}", file=sys.stderr)
-            if len(url_list) > 5:
-                rprint(f"    ... and {len(url_list) - 5} more", file=sys.stderr)
-
-        return 0
-
-    except Exception as e:
-        rprint(f"[red]Error creating crawl: {e}[/red]", file=sys.stderr)
-        return 1
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_crawls(
-    status: str | None = None,
-    urls__icontains: str | None = None,
-    max_depth: int | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Crawls as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.crawls.models import Crawl
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Crawl.objects.all().order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "urls__icontains": urls__icontains,
-        "max_depth": max_depth,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for crawl in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(crawl.status, "dim")
-            url_preview = crawl.urls[:50].replace("\n", " ")
-            rprint(f"[{status_color}]{crawl.status:8}[/{status_color}] [dim]{crawl.id}[/dim] {url_preview}...")
-        else:
-            write_record(crawl.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} crawls[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_crawls(
-    status: str | None = None,
-    max_depth: int | None = None,
-) -> int:
-    """
-    Update Crawls from stdin JSONL.
-
-    Reads Crawl records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from django.utils import timezone
-
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.crawls.models import Crawl
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        crawl_id = record.get("id")
-        if not crawl_id:
-            continue
-
-        try:
-            crawl = Crawl.objects.get(id=crawl_id)
-
-            # Apply updates from CLI flags
-            if status:
-                crawl.status = status
-                crawl.retry_at = timezone.now()
-            if max_depth is not None:
-                crawl.max_depth = max_depth
-
-            crawl.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(crawl.to_json())
-
-        except Crawl.DoesNotExist:
-            rprint(f"[yellow]Crawl not found: {crawl_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} crawls[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_crawls(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Crawls from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.crawls.models import Crawl
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    crawl_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not crawl_ids:
-        rprint("[yellow]No valid crawl IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    crawls = Crawl.objects.filter(id__in=crawl_ids)
-    count = crawls.count()
-
-    if count == 0:
-        rprint("[yellow]No matching crawls found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} crawls (dry run)[/yellow]", file=sys.stderr)
-        for crawl in crawls:
-            url_preview = crawl.urls[:50].replace("\n", " ")
-            rprint(f"  [dim]{crawl.id}[/dim] {url_preview}...", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = crawls.delete()
-    rprint(f"[green]Deleted {deleted_count} crawls[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Crawl records."""
-    pass
-
-
-@main.command("create")
-@click.argument("urls", nargs=-1)
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(urls: tuple, depth: int, tag: str, status: str):
-    """Create a Crawl job from URLs or stdin."""
-    sys.exit(create_crawl(urls, depth=depth, tag=tag, status=status))
-
-
-@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--urls__icontains", help="Filter by URLs contains")
-@click.option("--max-depth", type=int, help="Filter by max depth")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(
-    status: str | None,
-    urls__icontains: str | None,
-    max_depth: int | None,
-    limit: int | None,
-):
-    """List Crawls as JSONL."""
-    sys.exit(
-        list_crawls(
-            status=status,
-            urls__icontains=urls__icontains,
-            max_depth=max_depth,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--status", "-s", help="Set status")
-@click.option("--max-depth", type=int, help="Set max depth")
-def update_cmd(status: str | None, max_depth: int | None):
-    """Update Crawls from stdin JSONL."""
-    sys.exit(update_crawls(status=status, max_depth=max_depth))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Crawls from stdin JSONL."""
-    sys.exit(delete_crawls(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_add import add
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
-@click.argument("urls", nargs=-1)
-def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
-    del status, wait
-    add(list(urls), depth=depth, tag=tag, index_only=True, bg=True)
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()
@@ -1,311 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox extract [snapshot_ids...] [--plugins=NAMES]
-
-Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL.
-
-Input formats:
-    - Snapshot UUIDs (one per line)
-    - JSONL: {"type": "Snapshot", "id": "...", "url": "..."}
-    - JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."}
-
-Output (JSONL):
-    {"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."}
-
-Examples:
-    # Extract specific snapshot
-    archivebox extract 01234567-89ab-cdef-0123-456789abcdef
-
-    # Pipe from snapshot command
-    archivebox snapshot https://example.com | archivebox extract
-
-    # Run specific plugins only
-    archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef
-
-    # Chain commands
-    archivebox crawl https://example.com | archivebox snapshot | archivebox extract
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox extract"
-
-import sys
-from collections import defaultdict
-
-import rich_click as click
-
-
-def process_archiveresult_by_id(archiveresult_id: str) -> int:
-    """
-    Re-run extraction for a single ArchiveResult by ID.
-
-    ArchiveResults are projected status rows, not queued work items. Re-running
-    a single result means resetting that row and queueing its parent snapshot
-    through the shared crawl runner with the corresponding plugin selected.
-    """
-    from rich import print as rprint
-    from django.utils import timezone
-    from archivebox.core.models import ArchiveResult
-    from archivebox.services.runner import run_crawl
-
-    try:
-        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
-    except ArchiveResult.DoesNotExist:
-        rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
-        return 1
-
-    rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
-
-    try:
-        archiveresult.reset_for_retry()
-        snapshot = archiveresult.snapshot
-        snapshot.status = snapshot.StatusChoices.QUEUED
-        snapshot.retry_at = timezone.now()
-        snapshot.save(update_fields=["status", "retry_at", "modified_at"])
-
-        crawl = snapshot.crawl
-        if crawl.status != crawl.StatusChoices.STARTED:
-            crawl.status = crawl.StatusChoices.QUEUED
-        crawl.retry_at = timezone.now()
-        crawl.save(update_fields=["status", "retry_at", "modified_at"])
-
-        run_crawl(str(crawl.id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
-        archiveresult.refresh_from_db()
-
-        if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
-            print(f"[green]Extraction succeeded: {archiveresult.output_str}[/green]")
-            return 0
-        elif archiveresult.status == ArchiveResult.StatusChoices.NORESULTS:
-            print(f"[dim]Extraction completed with no results: {archiveresult.output_str}[/dim]")
-            return 0
-        elif archiveresult.status == ArchiveResult.StatusChoices.FAILED:
-            print(f"[red]Extraction failed: {archiveresult.output_str}[/red]", file=sys.stderr)
-            return 1
-        else:
-            # Still in progress or backoff - not a failure
-            print(f"[yellow]Extraction status: {archiveresult.status}[/yellow]")
-            return 0
-
-    except Exception as e:
-        print(f"[red]Extraction error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-        return 1
-
-
-def run_plugins(
-    args: tuple,
-    records: list[dict] | None = None,
-    plugins: str = "",
-    wait: bool = True,
-    emit_results: bool = True,
-) -> int:
-    """
-    Run plugins on Snapshots from input.
-
-    Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from rich import print as rprint
-    from django.utils import timezone
-
-    from archivebox.misc.jsonl import (
-        read_args_or_stdin,
-        write_record,
-        TYPE_SNAPSHOT,
-        TYPE_ARCHIVERESULT,
-    )
-    from archivebox.core.models import Snapshot
-    from archivebox.services.runner import run_crawl
-
-    is_tty = sys.stdout.isatty()
-
-    # Parse comma-separated plugins list once (reused in creation and filtering)
-    plugins_list = [p.strip() for p in plugins.split(",") if p.strip()] if plugins else []
-
-    # Parse stdin/args exactly once per CLI invocation.
-    # `main()` may already have consumed stdin to distinguish Snapshot input from
-    # ArchiveResult IDs; if so, it must pass the parsed records through here
-    # instead of asking this helper to reread an already-drained pipe.
-    if records is None:
-        records = list(read_args_or_stdin(args))
-
-    if not records:
-        rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
-        return 1
-
-    # Gather snapshot IDs and optional plugin constraints to process
-    snapshot_ids = set()
-    requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set)
-    for record in records:
-        record_type = record.get("type")
-
-        if record_type == TYPE_SNAPSHOT:
-            snapshot_id = record.get("id")
-            if snapshot_id:
-                snapshot_ids.add(snapshot_id)
-            elif record.get("url"):
-                # Look up by URL (get most recent if multiple exist)
-                snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
-                if snap:
-                    snapshot_ids.add(str(snap.id))
-                else:
-                    rprint(f"[yellow]Snapshot not found for URL: {record['url']}[/yellow]", file=sys.stderr)
-
-        elif record_type == TYPE_ARCHIVERESULT:
-            snapshot_id = record.get("snapshot_id")
-            if snapshot_id:
-                snapshot_ids.add(snapshot_id)
-                plugin_name = record.get("plugin")
-                if plugin_name and not plugins_list:
-                    requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
-
-        elif "id" in record:
-            # Assume it's a snapshot ID
-            snapshot_ids.add(record["id"])
-
-    if not snapshot_ids:
-        rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
-        return 1
-
-    # Get snapshots and ensure they have pending ArchiveResults
-    processed_count = 0
-    for snapshot_id in snapshot_ids:
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-        except Snapshot.DoesNotExist:
-            rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
-            continue
-
-        requested_plugin_names = set(plugins_list) | requested_plugins_by_snapshot.get(str(snapshot.id), set())
-        for plugin_name in requested_plugin_names:
-            existing_result = snapshot.archiveresult_set.filter(plugin=plugin_name).order_by("-created_at").first()
-            if existing_result:
-                existing_result.reset_for_retry()
-
-        # Reset snapshot status to allow processing
-        if snapshot.status == Snapshot.StatusChoices.SEALED:
-            snapshot.status = Snapshot.StatusChoices.STARTED
-            snapshot.retry_at = timezone.now()
-            snapshot.save()
-
-        processed_count += 1
-
-    if processed_count == 0:
-        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
-        return 1
-
-    rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
-
-    # Run orchestrator if --wait (default)
-    if wait:
-        rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
-        snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id in snapshot_ids:
-            try:
-                snapshot = Snapshot.objects.only("id", "crawl_id").get(id=snapshot_id)
-            except Snapshot.DoesNotExist:
-                continue
-            snapshot_ids_by_crawl[str(snapshot.crawl_id)].add(str(snapshot.id))
-
-        for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
-            selected_plugins = (
-                plugins_list
-                or sorted(
-                    {plugin for snapshot_id in crawl_snapshot_ids for plugin in requested_plugins_by_snapshot.get(str(snapshot_id), set())},
-                )
-                or None
-            )
-            run_crawl(
-                crawl_id,
-                snapshot_ids=sorted(crawl_snapshot_ids),
-                selected_plugins=selected_plugins,
-            )
-
-    if not emit_results:
-        return 0
-
-    # Output results as JSONL (when piped) or human-readable (when TTY)
-    for snapshot_id in snapshot_ids:
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-            results = snapshot.archiveresult_set.all()
-            if plugins_list:
-                results = results.filter(plugin__in=plugins_list)
-
-            for result in results:
-                if is_tty:
-                    status_color = {
-                        "succeeded": "green",
-                        "failed": "red",
-                        "skipped": "yellow",
-                    }.get(result.status, "dim")
-                    rprint(
-                        f"  [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ''}",
-                        file=sys.stderr,
-                    )
-                else:
-                    write_record(result.to_json())
-        except Snapshot.DoesNotExist:
-            continue
-
-    return 0
-
-
-def is_archiveresult_id(value: str) -> bool:
-    """Check if value looks like an ArchiveResult UUID."""
-    import re
-
-    uuid_pattern = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
-    if not uuid_pattern.match(value):
-        return False
-    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
-    from archivebox.core.models import ArchiveResult
-
-    return ArchiveResult.objects.filter(id=value).exists()
-
-
-@click.command()
-@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run (e.g., screenshot,singlefile)")
-@click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
-@click.argument("args", nargs=-1)
-def main(plugins: str, wait: bool, args: tuple):
-    """Run plugins on Snapshots, or process existing ArchiveResults by ID"""
-    from archivebox.misc.jsonl import read_args_or_stdin
-
-    # Read all input
-    records = list(read_args_or_stdin(args))
-
-    if not records:
-        from rich import print as rprint
-
-        rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
-        sys.exit(1)
-
-    # Check if input looks like existing ArchiveResult IDs to process
-    all_are_archiveresult_ids = all(is_archiveresult_id(r.get("id") or r.get("url", "")) for r in records)
-
-    if all_are_archiveresult_ids:
-        # Process existing ArchiveResults by ID
-        from rich import print as rprint
-
-        exit_code = 0
-        for record in records:
-            archiveresult_id = record.get("id") or record.get("url")
-            if not isinstance(archiveresult_id, str):
-                rprint(f"[red]Invalid ArchiveResult input: {record}[/red]", file=sys.stderr)
-                exit_code = 1
-                continue
-            result = process_archiveresult_by_id(archiveresult_id)
-            if result != 0:
-                exit_code = result
-        sys.exit(exit_code)
-    else:
-        # Default behavior: run plugins on Snapshots from input
-        sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,131 +1,32 @@
 #!/usr/bin/env python3
-__package__ = "archivebox.cli"
-__command__ = "archivebox help"

-import os
-from pathlib import Path
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox help'

-import click
-from rich import print
-from rich.panel import Panel
+import sys
+import argparse
+
+from typing import Optional, List, IO
+
+from ..main import help
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin


-def help() -> None:
-    """Print the ArchiveBox help message and usage"""
-
-    from archivebox.cli import ArchiveBoxGroup
-    from archivebox.config import CONSTANTS
-    from archivebox.config.permissions import IN_DOCKER
-    from archivebox.misc.logging_util import log_cli_command
-
-    log_cli_command("help", [], None, ".")
-
-    COMMANDS_HELP_TEXT = (
-        "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.meta_commands.keys()
-        )
-        + "\n\n    "
-        + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.setup_commands.keys()
-        )
-        + "\n\n    "
-        + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.archive_commands.keys()
-        )
+@docstring(help.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=help.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
    )
-
-    DOCKER_USAGE = (
-        """
-[dodger_blue3]Docker Usage:[/dodger_blue3]
-    [grey53]# using Docker Compose:[/grey53]
-    [blue]docker compose run[/blue] [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-
-    [grey53]# using Docker:[/grey53]
-    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 8000:8000[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-"""
-        if IN_DOCKER
-        else ""
-    )
-    DOCKER_DOCS = (
-        "\n    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker[/link]"
-        if IN_DOCKER
-        else ""
-    )
-    DOCKER_OUTSIDE_HINT = "\n    [grey53]# outside of Docker:[/grey53]" if IN_DOCKER else ""
-    DOCKER_CMD_PREFIX = "[blue]docker ... [/blue]" if IN_DOCKER else ""
-
-    print(f"""{DOCKER_USAGE}
-[deep_sky_blue4]Usage:[/deep_sky_blue4]{DOCKER_OUTSIDE_HINT}
-    [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-
-[deep_sky_blue4]Commands:[/deep_sky_blue4]
-    {COMMANDS_HELP_TEXT}
-
-[deep_sky_blue4]Documentation:[/deep_sky_blue4]
-    [link=https://github.com/ArchiveBox/ArchiveBox/wiki]https://github.com/ArchiveBox/ArchiveBox/wiki[/link]{DOCKER_DOCS}
-    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage[/link]
-    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
-""")
-
-    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
-        pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
-        EXAMPLE_USAGE = f"""
-[light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]
-
-[violet]Hint:[/violet] [i]Common maintenance tasks:[/i]
-    [dark_green]archivebox[/dark_green] [green]init[/green]      [grey53]# make sure database is up-to-date (safe to run multiple times)[/grey53]
-    [dark_green]archivebox[/dark_green] [green]install[/green]   [grey53]# make sure plugins are up-to-date (wget, chrome, singlefile, etc.)[/grey53]
-    [dark_green]archivebox[/dark_green] [green]status[/green]    [grey53]# get a health checkup report on your collection[/grey53]
-    [dark_green]archivebox[/dark_green] [green]update[/green]    [grey53]# retry any previously failed or interrupted archiving tasks[/grey53]
-
-[violet]Hint:[/violet] [i]More example usage:[/i]
-    [dark_green]archivebox[/dark_green] [green]add[/green] --depth=1 "https://example.com/some/page"
-    [dark_green]archivebox[/dark_green] [green]list[/green] --sort=timestamp --csv=timestamp,downloaded_at,url,title
-    [dark_green]archivebox[/dark_green] [green]schedule[/green] --every=day --depth=1 "https://example.com/some/feed.rss"
-    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:8000[/blue]                [grey53]# Start the Web UI / API server[/grey53]
-"""
-        print(
-            Panel(
-                EXAMPLE_USAGE,
-                expand=False,
-                border_style="grey53",
-                title="[green3]:white_check_mark: A collection [light_slate_blue]DATA DIR[/light_slate_blue] is currently active[/green3]",
-                subtitle="Commands run inside this dir will only apply to this collection.",
-            ),
-        )
-    else:
-        DATA_SETUP_HELP = "\n"
-        if IN_DOCKER:
-            DATA_SETUP_HELP += "[violet]Hint:[/violet] When using Docker, you need to mount a volume to use as your data dir:\n"
-            DATA_SETUP_HELP += "    docker run [violet]-v /some/path/data:/data[/violet] archivebox/archivebox ...\n\n"
-        DATA_SETUP_HELP += "To load an [dark_blue]existing[/dark_blue] collection:\n"
-        DATA_SETUP_HELP += "    1. [green]cd[/green] ~/archivebox/data     [grey53]# go into existing [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
-        DATA_SETUP_HELP += f"    2. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# migrate to latest version (safe to run multiple times)[/grey53]\n"
-        DATA_SETUP_HELP += f"    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-update all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
-        DATA_SETUP_HELP += f"    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ...get help with next steps... [/grey53]\n\n"
-        DATA_SETUP_HELP += "To start a [sea_green1]new[/sea_green1] collection:\n"
-        DATA_SETUP_HELP += "    1. [green]mkdir[/green] ~/archivebox/data  [grey53]# create a new, empty [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
-        DATA_SETUP_HELP += "    2. [green]cd[/green] ~/archivebox/data     [grey53]# cd into the new directory[/grey53]\n"
-        DATA_SETUP_HELP += f"    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# initialize ArchiveBox in the new data dir[/grey53]\n"
-        DATA_SETUP_HELP += f"    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-install all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
-        DATA_SETUP_HELP += f"    5. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ... get help with next steps... [/grey53]\n"
-        print(
-            Panel(
-                DATA_SETUP_HELP,
-                expand=False,
-                border_style="grey53",
-                title="[red]:cross_mark: No collection is currently active[/red]",
-                subtitle="All archivebox [green]commands[/green] should be run from inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
-            ),
-        )
+    parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)
+    
+    help(out_dir=pwd or OUTPUT_DIR)


-@click.command()
-@click.option("--help", "-h", is_flag=True, help="Show help")
-def main(**kwargs):
-    """Print the ArchiveBox help message and usage"""
-    return help()
-
-
-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,233 +1,52 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox init'

-import os
 import sys
-from pathlib import Path
-from collections.abc import Mapping
+import argparse

-from rich import print
-import rich_click as click
+from typing import Optional, List, IO

-from archivebox.misc.util import docstring, enforce_types
+from ..main import init
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin


-def _normalize_snapshot_record(link_dict: Mapping[str, object]) -> tuple[str, dict[str, object]] | None:
-    url = link_dict.get("url")
-    if not isinstance(url, str) or not url:
-        return None
-
-    record: dict[str, object] = {"url": url}
-    for key in ("timestamp", "title", "tags", "sources"):
-        value = link_dict.get(key)
-        if value is not None:
-            record[key] = value
-    return url, record
-
-
-@enforce_types
-def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
-    """Initialize a new ArchiveBox collection in the current directory"""
-
-    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
-    from archivebox.config.common import SERVER_CONFIG
-    from archivebox.config.collection import write_config_file
-    from archivebox.misc.legacy import parse_json_main_index, parse_json_links_details
-    from archivebox.misc.db import apply_migrations
-
-    # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
-    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
-    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
-
-    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
-    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
-    if is_empty and not existing_index:
-        print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
-        print("[green]----------------------------------------------------------------------[/green]")
-    elif existing_index:
-        # TODO: properly detect and print the existing version in current index as well
-        print(f"[green][*] Verifying and updating existing ArchiveBox collection to v{VERSION}...[/green]")
-        print("[green]----------------------------------------------------------------------[/green]")
-    else:
-        if force:
-            print("[red][!] This folder appears to already have files in it, but no index.sqlite3 is present.[/red]")
-            print("[red]    Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).[/red]")
-        else:
-            print(
-                "[red][X] This folder appears to already have files in it, but no index.sqlite3 present.[/red]\n\n"
-                "    You must run init in a completely empty directory, or an existing data folder.\n\n"
-                "    [violet]Hint:[/violet] To import an existing data folder make sure to cd into the folder first, \n"
-                "    then run and run 'archivebox init' to pick up where you left off.\n\n"
-                "    (Always make sure your data folder is backed up first before updating ArchiveBox)",
-            )
-            raise SystemExit(2)
-
-    if existing_index:
-        print("\n[green][*] Verifying archive folder structure...[/green]")
-    else:
-        print("\n[green][+] Building archive folder structure...[/green]")
-
-    print(
-        f"    + ./{CONSTANTS.ARCHIVE_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.SOURCES_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.LOGS_DIR.relative_to(DATA_DIR)}...",
-    )
-    Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
-    Path(CONSTANTS.ARCHIVE_DIR).mkdir(exist_ok=True)
-    Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
-
-    print(f"    + ./{CONSTANTS.CONFIG_FILE.relative_to(DATA_DIR)}...")
-
-    # create the .archivebox_id file with a unique ID for this collection
-    from archivebox.config.paths import _get_collection_id
-
-    _get_collection_id(DATA_DIR, force_create=True)
-
-    # create the ArchiveBox.conf file
-    write_config_file({"SECRET_KEY": SERVER_CONFIG.SECRET_KEY})
-
-    if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
-        print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]")
-    else:
-        print("\n[green][+] Building main SQL index and running initial migrations...[/green]")
-
-    from archivebox.config.django import setup_django
-
-    setup_django()
-
-    for migration_line in apply_migrations(DATA_DIR):
-        sys.stdout.write(f"    {migration_line}\n")
-
-    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
-    print()
-    print(f"    √ ./{CONSTANTS.DATABASE_FILE.relative_to(DATA_DIR)}")
-
-    # from django.contrib.auth.models import User
-    # if SHELL_CONFIG.IS_TTY and not User.objects.filter(is_superuser=True).exclude(username='system').exists():
-    #     print('{green}[+] Creating admin user account...{reset}'.format(**SHELL_CONFIG.ANSI))
-    #     call_command("createsuperuser", interactive=True)
-
-    print()
-    print("[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]")
-
-    from archivebox.core.models import Snapshot
-
-    all_links = Snapshot.objects.none()
-    pending_links: dict[str, dict[str, object]] = {}
-
-    if existing_index:
-        all_links = Snapshot.objects.all()
-        print(f"    √ Loaded {all_links.count()} links from existing main index.")
-
-    if quick:
-        print("    > Skipping orphan snapshot import (quick mode)")
-    else:
-        try:
-            # Import orphaned links from legacy JSON indexes
-            orphaned_json_links: dict[str, dict[str, object]] = {}
-            for link_dict in parse_json_main_index(DATA_DIR):
-                normalized = _normalize_snapshot_record(link_dict)
-                if normalized is None:
-                    continue
-                url, record = normalized
-                if not all_links.filter(url=url).exists():
-                    orphaned_json_links[url] = record
-            if orphaned_json_links:
-                pending_links.update(orphaned_json_links)
-                print(f"    [yellow]√ Added {len(orphaned_json_links)} orphaned links from existing JSON index...[/yellow]")
-
-            orphaned_data_dir_links: dict[str, dict[str, object]] = {}
-            for link_dict in parse_json_links_details(DATA_DIR):
-                normalized = _normalize_snapshot_record(link_dict)
-                if normalized is None:
-                    continue
-                url, record = normalized
-                if not all_links.filter(url=url).exists():
-                    orphaned_data_dir_links[url] = record
-            if orphaned_data_dir_links:
-                pending_links.update(orphaned_data_dir_links)
-                print(f"    [yellow]√ Added {len(orphaned_data_dir_links)} orphaned links from existing archive directories.[/yellow]")
-
-            if pending_links:
-                for link_dict in pending_links.values():
-                    Snapshot.from_json(link_dict)
-
-            # Hint for orphaned snapshot directories
-            print()
-            print("    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
-            print("        archivebox update")
-
-        except (KeyboardInterrupt, SystemExit):
-            print(file=sys.stderr)
-            print("[yellow]:stop_sign: Stopped checking archive directories due to Ctrl-C/SIGTERM[/yellow]", file=sys.stderr)
-            print("    Your archive data is safe, but you should re-run `archivebox init` to finish the process later.", file=sys.stderr)
-            print(file=sys.stderr)
-            print("    [violet]Hint:[/violet] In the future you can run a quick init without checking dirs like so:", file=sys.stderr)
-            print("        archivebox init --quick", file=sys.stderr)
-            raise SystemExit(1)
-
-    print("\n[green]----------------------------------------------------------------------[/green]")
-
-    from django.contrib.auth.models import User
-
-    if (SERVER_CONFIG.ADMIN_USERNAME and SERVER_CONFIG.ADMIN_PASSWORD) and not User.objects.filter(
-        username=SERVER_CONFIG.ADMIN_USERNAME,
-    ).exists():
-        print("[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]")
-        User.objects.create_superuser(username=SERVER_CONFIG.ADMIN_USERNAME, password=SERVER_CONFIG.ADMIN_PASSWORD)
-
-    if existing_index:
-        print("[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]")
-    else:
-        print(f"[green][√] Done. A new ArchiveBox collection was initialized ({len(all_links) + len(pending_links)} links).[/green]")
-
-    CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
-    CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
-    CONSTANTS.DEFAULT_LIB_DIR.mkdir(parents=True, exist_ok=True)
-    (CONSTANTS.DEFAULT_LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
-
-    from archivebox.config.common import STORAGE_CONFIG
-    from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir
-
-    STORAGE_CONFIG.TMP_DIR.mkdir(parents=True, exist_ok=True)
-    STORAGE_CONFIG.LIB_DIR.mkdir(parents=True, exist_ok=True)
-    (STORAGE_CONFIG.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
-
-    working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
-    if working_tmp_dir:
-        working_tmp_dir.mkdir(parents=True, exist_ok=True)
-
-    working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
-    if working_lib_dir:
-        working_lib_dir.mkdir(parents=True, exist_ok=True)
-        (working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)
-
-    if install:
-        from archivebox.cli.archivebox_install import install as install_method
-
-        install_method()
-
-    if Snapshot.objects.count() < 25:  # hide the hints for experienced users
-        print()
-        print("    [violet]Hint:[/violet] To view your archive index, run:")
-        print(
-            "        archivebox server  # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]",
-        )
-        print()
-        print("    To add new links, you can run:")
-        print("        archivebox add < ~/some/path/to/list_of_links.txt")
-        print()
-        print("    For more usage and examples, run:")
-        print("        archivebox help")
-
-
-@click.command()
-@click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
-@click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
-@click.option("--install", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
@docstring(init.__doc__)
-def main(**kwargs) -> None:
-    init(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=init.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--force', # '-f',
+        action='store_true',
+        help='Ignore unrecognized files in current directory and initialize anyway',
+    )
+    parser.add_argument(
+        '--quick', '-q',
+        action='store_true',
+        help='Run any updates or migrations without rechecking all snapshot dirs',
+    )
+    parser.add_argument(
+        '--setup', #'-s',
+        action='store_true',
+        help='Automatically install dependencies and extras used for archiving',
+    )
+    command = parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)

+    init(
+        force=command.force,
+        quick=command.quick,
+        setup=command.setup,
+        out_dir=pwd or OUTPUT_DIR,
+    )
+    

-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,104 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-
-import os
-
-import rich_click as click
-from rich import print
-
-from archivebox.misc.util import docstring, enforce_types
-
-
-@enforce_types
-def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
-    """Detect and install ArchiveBox dependencies by running the abx-dl install flow
-
-    Examples:
-        archivebox install                              # Install all dependencies
-        archivebox install wget curl                    # Install only wget and curl
-        archivebox install --binproviders=pip yt-dlp    # Install yt-dlp using only pip
-        archivebox install --binproviders=brew,apt      # Install all deps using only brew or apt
-    """
-
-    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
-    from archivebox.config.paths import ARCHIVE_DIR
-    from archivebox.misc.logging import stderr
-    from archivebox.cli.archivebox_init import init
-
-    if not (os.access(ARCHIVE_DIR, os.R_OK) and ARCHIVE_DIR.is_dir()):
-        init()  # must init full index because we need a db to store Binary entries in
-
-    # Show what we're installing
-    if binaries:
-        print(f"\n[green][+] Installing specific binaries: {', '.join(binaries)}[/green]")
-    else:
-        print("\n[green][+] Detecting and installing all ArchiveBox dependencies...[/green]")
-
-    if binproviders != "*":
-        print(f"[green][+] Using providers: {binproviders}[/green]")
-
-    if IS_ROOT:
-        EUID = os.geteuid()
-        print()
-        print(f"[yellow]:warning:  Running as UID=[blue]{EUID}[/blue].[/yellow]")
-        print(f"    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].")
-        print()
-
-    if dry_run:
-        print("[dim]Dry run - would run the abx-dl install flow[/dim]")
-        return
-
-    # Set up Django
-    from archivebox.config.django import setup_django
-
-    setup_django()
-
-    plugin_names = list(binaries)
-    if binproviders != "*":
-        plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
-
-    print("[+] Running installer via abx-dl bus...")
-    print()
-
-    from archivebox.services.runner import run_install
-
-    run_install(plugin_names=plugin_names or None)
-
-    print()
-
-    # Check for superuser
-    from django.contrib.auth import get_user_model
-
-    User = get_user_model()
-
-    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
-        stderr("\n[+] Don't forget to create a new admin user for the Web UI...", color="green")
-        stderr("    archivebox manage createsuperuser")
-
-    print()
-
-    # Show version to display full status including installed binaries
-    # Django is already loaded, so just import and call the function directly
-    from archivebox.cli.archivebox_version import version as show_version
-
-    show_version(quiet=False)
-
-
-@click.command()
-@click.argument("binaries", nargs=-1, type=str, required=False)
-@click.option(
-    "--binproviders",
-    "-p",
-    default="*",
-    help="Comma-separated list of providers to use (pip,npm,brew,apt,env,custom) or * for all",
-    show_default=True,
-)
-@click.option("--dry-run", "-d", is_flag=True, help="Show what would happen without actually running", default=False)
-@docstring(install.__doc__)
-def main(**kwargs) -> None:
-    install(**kwargs)
-
-
-if __name__ == "__main__":
-    main()
@@ -1,57 +1,139 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
-__command__ = "archivebox list"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox list'

 import sys
+import argparse

-import rich_click as click
+from typing import Optional, List, IO

-from archivebox.cli.archivebox_snapshot import list_snapshots
+from ..main import list_all
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..index import (
+    LINK_FILTERS,
+    get_indexed_folders,
+    get_archived_folders,
+    get_unarchived_folders,
+    get_present_folders,
+    get_valid_folders,
+    get_invalid_folders,
+    get_duplicate_folders,
+    get_orphaned_folders,
+    get_corrupted_folders,
+    get_unrecognized_folders,
+)
+from ..logging_util import SmartFormatter, reject_stdin, stderr


-@click.command()
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def main(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-) -> None:
-    """List Snapshots."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
+@docstring(list_all.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=list_all.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--csv', #'-c',
+        type=str,
+        help="Print the output in CSV format with the given columns, e.g.: timestamp,url,extension",
+        default=None,
+    )
+    group.add_argument(
+        '--json', #'-j',
+        action='store_true',
+        help="Print the output in JSON format with all columns included",
+    )
+    group.add_argument(
+        '--html',
+        action='store_true',
+        help="Print the output in HTML format"
+    )
+    parser.add_argument(
+        '--with-headers',
+        action='store_true',
+        help='Include the headers in the output document' 
+    )
+    parser.add_argument(
+        '--sort', #'-s',
+        type=str,
+        help="List the links sorted using the given key, e.g. timestamp or updated",
+        default=None,
+    )
+    parser.add_argument(
+        '--before', #'-b',
+        type=float,
+        help="List only links bookmarked before (less than) the given timestamp",
+        default=None,
+    )
+    parser.add_argument(
+        '--after', #'-a',
+        type=float,
+        help="List only links bookmarked after (greater than or equal to) the given timestamp",
+        default=None,
+    )
+    parser.add_argument(
+        '--status',
+        type=str,
+        choices=('indexed', 'archived', 'unarchived', 'present', 'valid', 'invalid', 'duplicate', 'orphaned', 'corrupted', 'unrecognized'),
+        default='indexed',
+        help=(
+            'List only links or data directories that have the given status\n'
+            f'    indexed       {get_indexed_folders.__doc__} (the default)\n'
+            f'    archived      {get_archived_folders.__doc__}\n'
+            f'    unarchived    {get_unarchived_folders.__doc__}\n'
+            '\n'
+            f'    present       {get_present_folders.__doc__}\n'
+            f'    valid         {get_valid_folders.__doc__}\n'
+            f'    invalid       {get_invalid_folders.__doc__}\n'
+            '\n'
+            f'    duplicate     {get_duplicate_folders.__doc__}\n'
+            f'    orphaned      {get_orphaned_folders.__doc__}\n'
+            f'    corrupted     {get_corrupted_folders.__doc__}\n'
+            f'    unrecognized  {get_unrecognized_folders.__doc__}\n'
+        )
+    )
+    parser.add_argument(
+        '--filter-type', '-t',
+        type=str,
+        choices=(*LINK_FILTERS.keys(), 'search'),
+        default='exact',
+        help='Type of pattern matching to use when filtering URLs',
+    )
+    parser.add_argument(
+        'filter_patterns',
+        nargs='*',
+        type=str,
+        default=None,
+        help='List only URLs matching these filter patterns'
+    )
+    command = parser.parse_args(args or ())
+    reject_stdin(stdin)

+    if command.with_headers and not (command.json or command.html or command.csv):
+        stderr(
+            '[X] --with-headers can only be used with --json, --html or --csv options\n',
+            color='red',
+        )
+        raise SystemExit(2)

-if __name__ == "__main__":
-    main()
+    matching_folders = list_all(
+        filter_patterns=command.filter_patterns,
+        filter_type=command.filter_type,
+        status=command.status,
+        after=command.after,
+        before=command.before,
+        sort=command.sort,
+        csv=command.csv,
+        json=command.json,
+        html=command.html,
+        with_headers=command.with_headers,
+        out_dir=pwd or OUTPUT_DIR,
+    )
+    raise SystemExit(not matching_folders)
+
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,102 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox machine <action> [--filters]
-
-Manage Machine records (system-managed, mostly read-only).
-
-Machine records track the host machines where ArchiveBox runs.
-They are created automatically by the system and are primarily for debugging.
-
-Actions:
-    list    - List Machines as JSONL (with optional filters)
-
-Examples:
-    # List all machines
-    archivebox machine list
-
-    # List machines by hostname
-    archivebox machine list --hostname__icontains=myserver
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox machine"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_utils import apply_filters
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_machines(
-    hostname__icontains: str | None = None,
-    os_platform: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Machines as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Machine
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Machine.objects.all().order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "hostname__icontains": hostname__icontains,
-        "os_platform": os_platform,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for machine in queryset:
-        if is_tty:
-            rprint(f"[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}")
-        else:
-            write_record(machine.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} machines[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Machine records (read-only, system-managed)."""
-    pass
-
-
-@main.command("list")
-@click.option("--hostname__icontains", help="Filter by hostname contains")
-@click.option("--os-platform", help="Filter by OS platform")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(hostname__icontains: str | None, os_platform: str | None, limit: int | None):
-    """List Machines as JSONL."""
-    sys.exit(
-        list_machines(
-            hostname__icontains=hostname__icontains,
-            os_platform=os_platform,
-            limit=limit,
-        ),
-    )
-
-
-if __name__ == "__main__":
-    main()
@@ -1,34 +1,24 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox manage'

-import rich_click as click
-from archivebox.misc.util import docstring, enforce_types
+import sys
+
+from typing import Optional, List, IO
+
+from ..main import manage
+from ..util import docstring
+from ..config import OUTPUT_DIR


-@enforce_types
-def manage(args: list[str] | None = None) -> None:
-    """Run an ArchiveBox Django management command"""
-
-    from archivebox.config.common import SHELL_CONFIG
-    from archivebox.misc.logging import stderr
-
-    if (args and "createsuperuser" in args) and (SHELL_CONFIG.IN_DOCKER and not SHELL_CONFIG.IS_TTY):
-        stderr("[!] Warning: you need to pass -it to use interactive commands in docker", color="lightyellow")
-        stderr("    docker run -it archivebox manage {}".format(" ".join(args or ["..."])), color="lightyellow")
-        stderr("")
-
-    from django.core.management import execute_from_command_line
-
-    execute_from_command_line(["manage.py", *(args or ["help"])])
-
-
-@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
-@click.argument("args", nargs=-1)
@docstring(manage.__doc__)
-def main(args: list[str] | None = None) -> None:
-    manage(args=args)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    manage(
+        args=args,
+        out_dir=pwd or OUTPUT_DIR,
+    )


-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,49 +0,0 @@
-#!/usr/bin/env python3
-"""
-archivebox mcp
-
-Start the Model Context Protocol (MCP) server in stdio mode.
-Exposes all ArchiveBox CLI commands as MCP tools for AI agents.
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox mcp"
-
-import rich_click as click
-
-from archivebox.misc.util import docstring, enforce_types
-
-
-@enforce_types
-def mcp():
-    """
-    Start the MCP server in stdio mode for AI agent control.
-
-    The MCP (Model Context Protocol) server exposes all ArchiveBox CLI commands
-    as tools that AI agents can discover and execute. It communicates via JSON-RPC
-    2.0 over stdin/stdout.
-
-    Example usage with an MCP client:
-        archivebox mcp < requests.jsonl > responses.jsonl
-
-    Or interactively:
-        archivebox mcp
-        {"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}
-        {"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}
-    """
-
-    from archivebox.mcp.server import run_mcp_server
-
-    # Run the stdio server (blocks until stdin closes)
-    run_mcp_server()
-
-
-@click.command()
-@docstring(mcp.__doc__)
-def main(**kwargs):
-    """Start the MCP server in stdio mode"""
-    mcp()
-
-
-if __name__ == "__main__":
-    main()
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox oneshot'
+
+import sys
+import argparse
+
+from pathlib import Path
+from typing import List, Optional, IO
+
+from ..main import oneshot
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, accept_stdin, stderr
+
+
+@docstring(oneshot.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=oneshot.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        'url',
+        type=str,
+        default=None,
+        help=(
+            'URLs or paths to archive e.g.:\n'
+            '    https://getpocket.com/users/USERNAME/feed/all\n'
+            '    https://example.com/some/rss/feed.xml\n'
+            '    https://example.com\n'
+            '    ~/Downloads/firefox_bookmarks_export.html\n'
+            '    ~/Desktop/sites_list.csv\n'
+        )
+    )
+    parser.add_argument(
+        "--extract",
+        type=str,
+        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
+              This does not take precedence over the configuration",
+        default=""
+    )
+    parser.add_argument(
+        '--out-dir',
+        type=str,
+        default=OUTPUT_DIR,
+        help= "Path to save the single archive folder to, e.g. ./example.com_archive"
+    )
+    command = parser.parse_args(args or ())
+    stdin_url = None
+    url = command.url
+    if not url:
+        stdin_url = accept_stdin(stdin)
+
+    if (stdin_url and url) or (not stdin and not url):
+        stderr(
+            '[X] You must pass a URL/path to add via stdin or CLI arguments.\n',
+            color='red',
+        )
+        raise SystemExit(2)
+    
+    oneshot(
+        url=stdin_url or url,
+        out_dir=Path(command.out_dir).resolve(),
+        extractors=command.extract,
+    )
+
+
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,776 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox persona <action> [args...] [--filters]
-
-Manage Persona records (browser profiles for archiving).
-
-Actions:
-    create  - Create Personas
-    list    - List Personas as JSONL (with optional filters)
-    update  - Update Personas from stdin JSONL
-    delete  - Delete Personas from stdin JSONL
-
-Examples:
-    # Create a new persona
-    archivebox persona create work
-    archivebox persona create --import=chrome personal
-    archivebox persona create --import=edge work
-
-    # List all personas
-    archivebox persona list
-
-    # Delete a persona
-    archivebox persona list --name=old | archivebox persona delete --yes
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox persona"
-
-import os
-import sys
-import shutil
-import platform
-import subprocess
-import tempfile
-import json
-from pathlib import Path
-from collections.abc import Iterable
-from collections import OrderedDict
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_utils import apply_filters
-from archivebox.personas import importers as persona_importers
-
-
-# =============================================================================
-# Browser Profile Locations
-# =============================================================================
-
-
-def get_chrome_user_data_dir() -> Path | None:
-    """Get the default Chrome user data directory for the current platform."""
-    system = platform.system()
-    home = Path.home()
-
-    if system == "Darwin":  # macOS
-        candidates = [
-            home / "Library" / "Application Support" / "Google" / "Chrome",
-            home / "Library" / "Application Support" / "Chromium",
-        ]
-    elif system == "Linux":
-        candidates = [
-            home / ".config" / "google-chrome",
-            home / ".config" / "chromium",
-            home / ".config" / "chrome",
-            home / "snap" / "chromium" / "common" / "chromium",
-        ]
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = [
-            local_app_data / "Google" / "Chrome" / "User Data",
-            local_app_data / "Chromium" / "User Data",
-        ]
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate.exists() and (candidate / "Default").exists():
-            return candidate
-
-    return None
-
-
-def get_brave_user_data_dir() -> Path | None:
-    """Get the default Brave user data directory for the current platform."""
-    system = platform.system()
-    home = Path.home()
-
-    if system == "Darwin":
-        candidates = [
-            home / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser",
-        ]
-    elif system == "Linux":
-        candidates = [
-            home / ".config" / "BraveSoftware" / "Brave-Browser",
-        ]
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = [
-            local_app_data / "BraveSoftware" / "Brave-Browser" / "User Data",
-        ]
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate.exists() and (candidate / "Default").exists():
-            return candidate
-
-    return None
-
-
-def get_edge_user_data_dir() -> Path | None:
-    """Get the default Edge user data directory for the current platform."""
-    system = platform.system()
-    home = Path.home()
-
-    if system == "Darwin":
-        candidates = [
-            home / "Library" / "Application Support" / "Microsoft Edge",
-        ]
-    elif system == "Linux":
-        candidates = [
-            home / ".config" / "microsoft-edge",
-            home / ".config" / "microsoft-edge-beta",
-            home / ".config" / "microsoft-edge-dev",
-        ]
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = [
-            local_app_data / "Microsoft" / "Edge" / "User Data",
-        ]
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate.exists() and (candidate / "Default").exists():
-            return candidate
-
-    return None
-
-
-def get_browser_binary(browser: str) -> str | None:
-    system = platform.system()
-    home = Path.home()
-    browser = browser.lower()
-
-    if system == "Darwin":
-        candidates = {
-            "chrome": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"],
-            "chromium": ["/Applications/Chromium.app/Contents/MacOS/Chromium"],
-            "brave": ["/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"],
-            "edge": ["/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"],
-        }.get(browser, [])
-    elif system == "Linux":
-        candidates = {
-            "chrome": [
-                "/usr/bin/google-chrome",
-                "/usr/bin/google-chrome-stable",
-                "/usr/bin/google-chrome-beta",
-                "/usr/bin/google-chrome-unstable",
-            ],
-            "chromium": ["/usr/bin/chromium", "/usr/bin/chromium-browser"],
-            "brave": ["/usr/bin/brave-browser", "/usr/bin/brave-browser-beta", "/usr/bin/brave-browser-nightly"],
-            "edge": [
-                "/usr/bin/microsoft-edge",
-                "/usr/bin/microsoft-edge-stable",
-                "/usr/bin/microsoft-edge-beta",
-                "/usr/bin/microsoft-edge-dev",
-            ],
-        }.get(browser, [])
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = {
-            "chrome": [
-                str(local_app_data / "Google" / "Chrome" / "Application" / "chrome.exe"),
-                "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
-                "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
-            ],
-            "chromium": [str(local_app_data / "Chromium" / "Application" / "chrome.exe")],
-            "brave": [
-                str(local_app_data / "BraveSoftware" / "Brave-Browser" / "Application" / "brave.exe"),
-                "C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
-                "C:\\Program Files (x86)\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
-            ],
-            "edge": [
-                str(local_app_data / "Microsoft" / "Edge" / "Application" / "msedge.exe"),
-                "C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
-                "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
-            ],
-        }.get(browser, [])
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate and Path(candidate).exists():
-            return candidate
-
-    return None
-
-
-BROWSER_PROFILE_FINDERS = {
-    "chrome": get_chrome_user_data_dir,
-    "chromium": get_chrome_user_data_dir,  # Same locations
-    "brave": get_brave_user_data_dir,
-    "edge": get_edge_user_data_dir,
-}
-
-CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"}
-
-
-# =============================================================================
-# Cookie Extraction via CDP
-# =============================================================================
-
-NETSCAPE_COOKIE_HEADER = [
-    "# Netscape HTTP Cookie File",
-    "# https://curl.se/docs/http-cookies.html",
-    "# This file was generated by ArchiveBox persona cookie extraction",
-    "#",
-    "# Format: domain\\tincludeSubdomains\\tpath\\tsecure\\texpiry\\tname\\tvalue",
-    "",
-]
-
-
-def _parse_netscape_cookies(path: Path) -> "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]":
-    cookies = OrderedDict()
-    if not path.exists():
-        return cookies
-
-    for line in path.read_text().splitlines():
-        if not line or line.startswith("#"):
-            continue
-        parts = line.split("\t")
-        if len(parts) < 7:
-            continue
-        domain, include_subdomains, cookie_path, secure, expiry, name, value = parts[:7]
-        key = (domain, cookie_path, name)
-        cookies[key] = (domain, include_subdomains, cookie_path, secure, expiry, name, value)
-    return cookies
-
-
-def _write_netscape_cookies(path: Path, cookies: "OrderedDict[tuple[str, str, str], tuple[str, str, str, str, str, str, str]]") -> None:
-    lines = list(NETSCAPE_COOKIE_HEADER)
-    for cookie in cookies.values():
-        lines.append("\t".join(cookie))
-    path.write_text("\n".join(lines) + "\n")
-
-
-def _merge_netscape_cookies(existing_file: Path, new_file: Path) -> None:
-    existing = _parse_netscape_cookies(existing_file)
-    new = _parse_netscape_cookies(new_file)
-    for key, cookie in new.items():
-        existing[key] = cookie
-    _write_netscape_cookies(existing_file, existing)
-
-
-def extract_cookies_via_cdp(
-    user_data_dir: Path,
-    output_file: Path,
-    profile_dir: str | None = None,
-    chrome_binary: str | None = None,
-) -> bool:
-    """
-    Launch Chrome with the given user data dir and extract cookies via CDP.
-
-    Returns True if successful, False otherwise.
-    """
-    from archivebox.config.common import STORAGE_CONFIG
-
-    # Find the cookie extraction script
-    chrome_plugin_dir = Path(__file__).parent.parent / "plugins" / "chrome"
-    extract_script = chrome_plugin_dir / "extract_cookies.js"
-
-    if not extract_script.exists():
-        rprint(f"[yellow]Cookie extraction script not found at {extract_script}[/yellow]", file=sys.stderr)
-        return False
-
-    # Get node modules dir
-    node_modules_dir = STORAGE_CONFIG.LIB_DIR / "npm" / "node_modules"
-
-    # Set up environment
-    env = os.environ.copy()
-    env["NODE_MODULES_DIR"] = str(node_modules_dir)
-    env["CHROME_USER_DATA_DIR"] = str(user_data_dir)
-    env["CHROME_HEADLESS"] = "true"
-    if chrome_binary:
-        env["CHROME_BINARY"] = str(chrome_binary)
-    output_path = output_file
-    temp_output = None
-    temp_dir = None
-    if output_file.exists():
-        temp_dir = Path(tempfile.mkdtemp(prefix="ab_cookies_"))
-        temp_output = temp_dir / "cookies.txt"
-        output_path = temp_output
-    if profile_dir:
-        extra_arg = f"--profile-directory={profile_dir}"
-        existing_extra = env.get("CHROME_ARGS_EXTRA", "").strip()
-        args_list = []
-        if existing_extra:
-            if existing_extra.startswith("["):
-                try:
-                    parsed = json.loads(existing_extra)
-                    if isinstance(parsed, list):
-                        args_list.extend(str(x) for x in parsed)
-                except Exception:
-                    args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
-            else:
-                args_list.extend([s.strip() for s in existing_extra.split(",") if s.strip()])
-        args_list.append(extra_arg)
-        env["CHROME_ARGS_EXTRA"] = json.dumps(args_list)
-
-    env["COOKIES_OUTPUT_FILE"] = str(output_path)
-
-    try:
-        result = subprocess.run(
-            ["node", str(extract_script)],
-            env=env,
-            capture_output=True,
-            text=True,
-            timeout=60,
-        )
-
-        if result.returncode == 0:
-            if temp_output and temp_output.exists():
-                _merge_netscape_cookies(output_file, temp_output)
-            return True
-        else:
-            rprint(f"[yellow]Cookie extraction failed: {result.stderr}[/yellow]", file=sys.stderr)
-            return False
-
-    except subprocess.TimeoutExpired:
-        rprint("[yellow]Cookie extraction timed out[/yellow]", file=sys.stderr)
-        return False
-    except FileNotFoundError:
-        rprint("[yellow]Node.js not found. Cannot extract cookies.[/yellow]", file=sys.stderr)
-        return False
-    except Exception as e:
-        rprint(f"[yellow]Cookie extraction error: {e}[/yellow]", file=sys.stderr)
-        return False
-    finally:
-        if temp_dir and temp_dir.exists():
-            shutil.rmtree(temp_dir, ignore_errors=True)
-
-
-# =============================================================================
-# Validation Helpers
-# =============================================================================
-
-
-def validate_persona_name(name: str) -> tuple[bool, str]:
-    """
-    Validate persona name to prevent path traversal attacks.
-
-    Returns:
-        (is_valid, error_message): tuple indicating if name is valid
-    """
-    if not name or not name.strip():
-        return False, "Persona name cannot be empty"
-
-    # Check for path separators
-    if "/" in name or "\\" in name:
-        return False, "Persona name cannot contain path separators (/ or \\)"
-
-    # Check for parent directory references
-    if ".." in name:
-        return False, "Persona name cannot contain parent directory references (..)"
-
-    # Check for hidden files/directories
-    if name.startswith("."):
-        return False, "Persona name cannot start with a dot (.)"
-
-    # Ensure name doesn't contain null bytes or other dangerous chars
-    if "\x00" in name or "\n" in name or "\r" in name:
-        return False, "Persona name contains invalid characters"
-
-    return True, ""
-
-
-def ensure_path_within_personas_dir(persona_path: Path) -> bool:
-    """
-    Verify that a persona path is within PERSONAS_DIR.
-
-    This is a safety check to prevent path traversal attacks where
-    a malicious persona name could cause operations on paths outside
-    the expected PERSONAS_DIR.
-
-    Returns:
-        True if path is safe, False otherwise
-    """
-    from archivebox.config.constants import CONSTANTS
-
-    try:
-        # Resolve both paths to absolute paths
-        personas_dir = CONSTANTS.PERSONAS_DIR.resolve()
-        resolved_path = persona_path.resolve()
-
-        # Check if resolved_path is a child of personas_dir
-        return resolved_path.is_relative_to(personas_dir)
-    except (ValueError, RuntimeError):
-        return False
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_personas(
-    names: Iterable[str],
-    import_from: str | None = None,
-    profile: str | None = None,
-) -> int:
-    """
-    Create Personas from names.
-
-    If --import is specified, copy the browser profile to the persona directory
-    and extract cookies.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.personas.models import Persona
-
-    is_tty = sys.stdout.isatty()
-    name_list = list(names) if names else []
-
-    if not name_list:
-        rprint("[yellow]No persona names provided. Pass names as arguments.[/yellow]", file=sys.stderr)
-        return 1
-
-    # Validate import source if specified
-    source_profile_dir = None
-    if import_from:
-        import_from = import_from.lower()
-        if import_from not in BROWSER_PROFILE_FINDERS:
-            rprint(f"[red]Unknown browser: {import_from}[/red]", file=sys.stderr)
-            rprint(f"[dim]Supported browsers: {', '.join(BROWSER_PROFILE_FINDERS.keys())}[/dim]", file=sys.stderr)
-            return 1
-
-        source_profile_dir = BROWSER_PROFILE_FINDERS[import_from]()
-        if not source_profile_dir:
-            rprint(f"[red]Could not find {import_from} profile directory[/red]", file=sys.stderr)
-            return 1
-
-        rprint(f"[dim]Found {import_from} profile: {source_profile_dir}[/dim]", file=sys.stderr)
-
-        if profile is None and (source_profile_dir / "Default").exists():
-            profile = "Default"
-
-        browser_binary = get_browser_binary(import_from)
-        if browser_binary:
-            rprint(f"[dim]Using {import_from} binary: {browser_binary}[/dim]", file=sys.stderr)
-
-    created_count = 0
-    for name in name_list:
-        name = name.strip()
-        if not name:
-            continue
-
-        # Validate persona name to prevent path traversal
-        is_valid, error_msg = persona_importers.validate_persona_name(name)
-        if not is_valid:
-            rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
-            continue
-
-        persona, created = Persona.objects.get_or_create(name=name)
-
-        if created:
-            persona.ensure_dirs()
-            created_count += 1
-            rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
-        else:
-            rprint(f"[dim]Persona already exists: {name}[/dim]", file=sys.stderr)
-
-        cookies_file = Path(persona.path) / "cookies.txt"
-
-        # Import browser profile if requested
-        if import_from in CHROMIUM_BROWSERS and source_profile_dir is not None:
-            try:
-                import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile)
-                import_result = persona_importers.import_persona_from_source(
-                    persona,
-                    import_source,
-                    copy_profile=True,
-                    import_cookies=True,
-                    capture_storage=False,
-                )
-            except Exception as e:
-                rprint(f"[red]Failed to import browser profile: {e}[/red]", file=sys.stderr)
-                return 1
-
-            if import_result.profile_copied:
-                rprint("[green]Copied browser profile to persona[/green]", file=sys.stderr)
-            if import_result.cookies_imported:
-                rprint(f"[green]Extracted cookies to {cookies_file}[/green]", file=sys.stderr)
-            elif not import_result.profile_copied:
-                rprint("[yellow]Could not import cookies automatically.[/yellow]", file=sys.stderr)
-
-            for warning in import_result.warnings:
-                rprint(f"[yellow]{warning}[/yellow]", file=sys.stderr)
-
-        if not is_tty:
-            write_record(
-                {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
-                    "name": persona.name,
-                    "path": str(persona.path),
-                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
-                    "COOKIES_FILE": persona.COOKIES_FILE,
-                },
-            )
-
-    rprint(f"[green]Created {created_count} new persona(s)[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_personas(
-    name: str | None = None,
-    name__icontains: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Personas as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.personas.models import Persona
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Persona.objects.all().order_by("name")
-
-    # Apply filters
-    filter_kwargs = {
-        "name": name,
-        "name__icontains": name__icontains,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for persona in queryset:
-        cookies_status = "[green]✓[/green]" if persona.COOKIES_FILE else "[dim]✗[/dim]"
-        chrome_status = "[green]✓[/green]" if Path(persona.CHROME_USER_DATA_DIR).exists() else "[dim]✗[/dim]"
-
-        if is_tty:
-            rprint(f"[cyan]{persona.name:20}[/cyan] cookies:{cookies_status} chrome:{chrome_status} [dim]{persona.path}[/dim]")
-        else:
-            write_record(
-                {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
-                    "name": persona.name,
-                    "path": str(persona.path),
-                    "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
-                    "COOKIES_FILE": persona.COOKIES_FILE,
-                },
-            )
-        count += 1
-
-    rprint(f"[dim]Listed {count} persona(s)[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_personas(name: str | None = None) -> int:
-    """
-    Update Personas from stdin JSONL.
-
-    Reads Persona records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.personas.models import Persona
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        persona_id = record.get("id")
-        old_name = record.get("name")
-
-        if not persona_id and not old_name:
-            continue
-
-        try:
-            if persona_id:
-                persona = Persona.objects.get(id=persona_id)
-            else:
-                persona = Persona.objects.get(name=old_name)
-
-            # Apply updates from CLI flags
-            if name:
-                # Validate new name to prevent path traversal
-                is_valid, error_msg = persona_importers.validate_persona_name(name)
-                if not is_valid:
-                    rprint(f'[red]Invalid new persona name "{name}": {error_msg}[/red]', file=sys.stderr)
-                    continue
-
-                # Rename the persona directory too
-                old_path = persona.path
-                persona.name = name
-                new_path = persona.path
-
-                if old_path.exists() and old_path != new_path:
-                    shutil.move(str(old_path), str(new_path))
-
-                persona.save()
-
-            updated_count += 1
-
-            if not is_tty:
-                write_record(
-                    {
-                        "id": str(persona.id) if hasattr(persona, "id") else None,
-                        "name": persona.name,
-                        "path": str(persona.path),
-                    },
-                )
-
-        except Persona.DoesNotExist:
-            rprint(f"[yellow]Persona not found: {persona_id or old_name}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} persona(s)[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_personas(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Personas from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.personas.models import Persona
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    # Collect persona IDs or names
-    persona_ids = []
-    persona_names = []
-    for r in records:
-        if r.get("id"):
-            persona_ids.append(r["id"])
-        elif r.get("name"):
-            persona_names.append(r["name"])
-
-    if not persona_ids and not persona_names:
-        rprint("[yellow]No valid persona IDs or names in input[/yellow]", file=sys.stderr)
-        return 1
-
-    from django.db.models import Q
-
-    query = Q()
-    if persona_ids:
-        query |= Q(id__in=persona_ids)
-    if persona_names:
-        query |= Q(name__in=persona_names)
-
-    personas = Persona.objects.filter(query)
-    count = personas.count()
-
-    if count == 0:
-        rprint("[yellow]No matching personas found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} persona(s) (dry run)[/yellow]", file=sys.stderr)
-        for persona in personas:
-            rprint(f"  {persona.name} ({persona.path})", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Delete persona directories and database records
-    deleted_count = 0
-    for persona in personas:
-        persona_path = persona.path
-
-        # Safety check: ensure path is within PERSONAS_DIR before deletion
-        if not ensure_path_within_personas_dir(persona_path):
-            rprint(f'[red]Security error: persona path "{persona_path}" is outside PERSONAS_DIR. Skipping deletion.[/red]', file=sys.stderr)
-            continue
-
-        if persona_path.exists():
-            shutil.rmtree(persona_path)
-        persona.delete()
-        deleted_count += 1
-
-    rprint(f"[green]Deleted {deleted_count} persona(s)[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Persona records (browser profiles)."""
-    pass
-
-
-@main.command("create")
-@click.argument("names", nargs=-1)
-@click.option("--import", "import_from", help="Import profile from browser (chrome, chromium, brave, edge)")
-@click.option("--profile", help="Profile directory name under the user data dir (e.g. Default, Profile 1)")
-def create_cmd(names: tuple, import_from: str | None, profile: str | None):
-    """Create Personas, optionally importing from a browser profile."""
-    sys.exit(create_personas(names, import_from=import_from, profile=profile))
-
-
-@main.command("list")
-@click.option("--name", help="Filter by exact name")
-@click.option("--name__icontains", help="Filter by name contains")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
-    """List Personas as JSONL."""
-    sys.exit(list_personas(name=name, name__icontains=name__icontains, limit=limit))
-
-
-@main.command("update")
-@click.option("--name", "-n", help="Set new name")
-def update_cmd(name: str | None):
-    """Update Personas from stdin JSONL."""
-    sys.exit(update_personas(name=name))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Personas from stdin JSONL."""
-    sys.exit(delete_personas(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,309 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-
-
-import rich_click as click
-
-from archivebox.misc.util import docstring, enforce_types
-
-
-# State Machine ASCII Art Diagrams
-CRAWL_MACHINE_DIAGRAM = """
-┌─────────────────────────────────────────────────────────────────────────────┐
-│                              CrawlMachine                                   │
-├─────────────────────────────────────────────────────────────────────────────┤
-│                                                                             │
-│   ┌─────────────┐                                                           │
-│   │   QUEUED    │◄────────────────┐                                         │
-│   │  (initial)  │                 │                                         │
-│   └──────┬──────┘                 │                                         │
-│          │                        │ tick() unless can_start()               │
-│          │ tick() when            │                                         │
-│          │ can_start()            │                                         │
-│          ▼                        │                                         │
-│   ┌─────────────┐                 │                                         │
-│   │   STARTED   │─────────────────┘                                         │
-│   │             │◄────────────────┐                                         │
-│   │ enter:      │                 │                                         │
-│   │  crawl.run()│                 │ tick() unless is_finished()             │
-│   │  (discover  │                 │                                         │
-│   │   Crawl     │─────────────────┘                                         │
-│   │   hooks)    │                                                           │
-│   └──────┬──────┘                                                           │
-│          │                                                                  │
-│          │ tick() when is_finished()                                        │
-│          ▼                                                                  │
-│   ┌─────────────┐                                                           │
-│   │   SEALED    │                                                           │
-│   │   (final)   │                                                           │
-│   │             │                                                           │
-│   │ enter:      │                                                           │
-│   │  cleanup()  │                                                           │
-│   └─────────────┘                                                           │
-│                                                                             │
-│   Hooks triggered: on_Crawl__* (during STARTED.enter via crawl.run())       │
-│                    on_CrawlEnd__* (during SEALED.enter via cleanup())       │
-└─────────────────────────────────────────────────────────────────────────────┘
-"""
-
-SNAPSHOT_MACHINE_DIAGRAM = """
-┌─────────────────────────────────────────────────────────────────────────────┐
-│                            SnapshotMachine                                  │
-├─────────────────────────────────────────────────────────────────────────────┤
-│                                                                             │
-│   ┌─────────────┐                                                           │
-│   │   QUEUED    │◄────────────────┐                                         │
-│   │  (initial)  │                 │                                         │
-│   └──────┬──────┘                 │                                         │
-│          │                        │ tick() unless can_start()               │
-│          │ tick() when            │                                         │
-│          │ can_start()            │                                         │
-│          ▼                        │                                         │
-│   ┌─────────────┐                 │                                         │
-│   │   STARTED   │─────────────────┘                                         │
-│   │             │◄────────────────┐                                         │
-│   │ enter:      │                 │                                         │
-│   │ snapshot    │                 │ tick() unless is_finished()             │
-│   │  .run()     │                 │                                         │
-│   │ (discover   │─────────────────┘                                         │
-│   │  Snapshot   │                                                           │
-│   │  hooks,     │                                                           │
-│   │  create     │                                                           │
-│   │  pending    │                                                           │
-│   │  results)   │                                                           │
-│   └──────┬──────┘                                                           │
-│          │                                                                  │
-│          │ tick() when is_finished()                                        │
-│          ▼                                                                  │
-│   ┌─────────────┐                                                           │
-│   │   SEALED    │                                                           │
-│   │   (final)   │                                                           │
-│   │             │                                                           │
-│   │ enter:      │                                                           │
-│   │  cleanup()  │                                                           │
-│   └─────────────┘                                                           │
-│                                                                             │
-│   Hooks triggered: on_Snapshot__* (creates ArchiveResults in STARTED.enter) │
-└─────────────────────────────────────────────────────────────────────────────┘
-"""
-
-BINARY_MACHINE_DIAGRAM = """
-┌─────────────────────────────────────────────────────────────────────────────┐
-│                             BinaryMachine                                   │
-├─────────────────────────────────────────────────────────────────────────────┤
-│                                                                             │
-│   ┌─────────────┐                                                           │
-│   │   QUEUED    │◄────────────────┐                                         │
-│   │  (initial)  │                 │                                         │
-│   └──────┬──────┘                 │                                         │
-│          │                        │ tick() unless can_install()             │
-│          │                        │ (stays queued if failed)                │
-│          │ tick() when            │                                         │
-│          │ can_install()          │                                         │
-│          │                        │                                         │
-│          │ on_install() runs      │                                         │
-│          │ during transition:     │                                         │
-│          │  • binary.run()        │                                         │
-│          │    (discover Binary    │                                         │
-│          │     hooks, try each    │                                         │
-│          │     provider until     │                                         │
-│          │     one succeeds)      │                                         │
-│          │  • Sets abspath,       │                                         │
-│          │    version, sha256     │                                         │
-│          │                        │                                         │
-│          │ If install fails:      │                                         │
-│          │  raises exception──────┘                                         │
-│          │  (retry_at bumped)                                               │
-│          │                                                                  │
-│          ▼                                                                  │
-│   ┌─────────────┐                                                           │
-│   │  INSTALLED  │                                                           │
-│   │   (final)   │                                                           │
-│   │             │                                                           │
-│   │ Binary is   │                                                           │
-│   │ ready to    │                                                           │
-│   │ use         │                                                           │
-│   └─────────────┘                                                           │
-│                                                                             │
-│   Hooks triggered: on_Binary__* (provider hooks during transition)          │
-│   Providers tried in sequence until one succeeds: apt, brew, pip, npm, etc. │
-│   Installation is synchronous - no intermediate STARTED state               │
-└─────────────────────────────────────────────────────────────────────────────┘
-"""
-
-
-@enforce_types
-def pluginmap(
-    show_disabled: bool = False,
-    model: str | None = None,
-    quiet: bool = False,
-) -> dict:
-    """
-    Show a map of all state machines and their associated plugin hooks.
-
-    Displays ASCII art diagrams of the core queued model state machines (Crawl,
-    Snapshot, Binary) and lists all auto-detected on_Modelname_xyz hooks
-    that will run for each model's transitions.
-    """
-    from rich.console import Console
-    from rich.table import Table
-    from rich.panel import Panel
-    from rich import box
-
-    from archivebox.hooks import (
-        discover_hooks,
-        is_background_hook,
-        BUILTIN_PLUGINS_DIR,
-        USER_PLUGINS_DIR,
-    )
-
-    console = Console()
-    prnt = console.print
-
-    # Model event types that can have hooks
-    model_events = {
-        "Crawl": {
-            "description": "Hooks run when a Crawl starts (QUEUED→STARTED)",
-            "machine": "CrawlMachine",
-            "diagram": CRAWL_MACHINE_DIAGRAM,
-        },
-        "CrawlEnd": {
-            "description": "Hooks run when a Crawl finishes (STARTED→SEALED)",
-            "machine": "CrawlMachine",
-            "diagram": None,  # Part of CrawlMachine
-        },
-        "Snapshot": {
-            "description": "Hooks run for each Snapshot (creates ArchiveResults)",
-            "machine": "SnapshotMachine",
-            "diagram": SNAPSHOT_MACHINE_DIAGRAM,
-        },
-        "Binary": {
-            "description": "Hooks for installing binary dependencies (providers)",
-            "machine": "BinaryMachine",
-            "diagram": BINARY_MACHINE_DIAGRAM,
-        },
-    }
-
-    # Filter to specific model if requested
-    if model:
-        model = model.title()
-        if model not in model_events:
-            prnt(f'[red]Error: Unknown model "{model}". Available: {", ".join(model_events.keys())}[/red]')
-            return {}
-        model_events = {model: model_events[model]}
-
-    result = {
-        "models": {},
-        "plugins_dir": str(BUILTIN_PLUGINS_DIR),
-        "user_plugins_dir": str(USER_PLUGINS_DIR),
-    }
-
-    if not quiet:
-        prnt()
-        prnt("[bold cyan]ArchiveBox Plugin Map[/bold cyan]")
-        prnt(f"[dim]Built-in plugins: {BUILTIN_PLUGINS_DIR}[/dim]")
-        prnt(f"[dim]User plugins: {USER_PLUGINS_DIR}[/dim]")
-        prnt()
-
-    for event_name, info in model_events.items():
-        # Discover hooks for this event
-        hooks = discover_hooks(event_name, filter_disabled=not show_disabled)
-
-        # Build hook info list
-        hook_infos = []
-        for hook_path in hooks:
-            # Get plugin name from parent directory (e.g., 'wget' from 'plugins/wget/on_Snapshot__06_wget.bg.py')
-            plugin_name = hook_path.parent.name
-            is_bg = is_background_hook(hook_path.name)
-
-            hook_infos.append(
-                {
-                    "path": str(hook_path),
-                    "name": hook_path.name,
-                    "plugin": plugin_name,
-                    "is_background": is_bg,
-                    "extension": hook_path.suffix,
-                },
-            )
-
-        result["models"][event_name] = {
-            "description": info["description"],
-            "machine": info["machine"],
-            "hooks": hook_infos,
-            "hook_count": len(hook_infos),
-        }
-
-        if not quiet:
-            # Show diagram if this model has one
-            if info.get("diagram"):
-                assert info["diagram"] is not None
-                prnt(
-                    Panel(
-                        info["diagram"],
-                        title=f"[bold green]{info['machine']}[/bold green]",
-                        border_style="green",
-                        expand=False,
-                    ),
-                )
-                prnt()
-
-            # Create hooks table
-            table = Table(
-                title=f"[bold yellow]on_{event_name}__* Hooks[/bold yellow] ({len(hooks)} found)",
-                box=box.ROUNDED,
-                show_header=True,
-                header_style="bold magenta",
-            )
-            table.add_column("Plugin", style="cyan", width=20)
-            table.add_column("Hook Name", style="green")
-            table.add_column("BG", justify="center", width=4)
-            table.add_column("Type", justify="center", width=5)
-
-            # Sort lexicographically by hook name
-            sorted_hooks = sorted(hook_infos, key=lambda h: h["name"])
-
-            for hook in sorted_hooks:
-                bg_marker = "[yellow]bg[/yellow]" if hook["is_background"] else ""
-                ext = hook["extension"].lstrip(".")
-                table.add_row(
-                    hook["plugin"],
-                    hook["name"],
-                    bg_marker,
-                    ext,
-                )
-
-            prnt(table)
-            prnt()
-            prnt(f"[dim]{info['description']}[/dim]")
-            prnt()
-
-    # Summary
-    if not quiet:
-        total_hooks = sum(m["hook_count"] for m in result["models"].values())
-        prnt(f"[bold]Total hooks discovered: {total_hooks}[/bold]")
-        prnt()
-        prnt("[dim]Hook naming convention: on_{Model}__{XX}_{description}[.bg].{ext}[/dim]")
-        prnt("[dim]  - XX: Two-digit lexicographic order (00-99)[/dim]")
-        prnt("[dim]  - .bg: Background hook (non-blocking)[/dim]")
-        prnt("[dim]  - ext: py, sh, or js[/dim]")
-        prnt()
-
-    return result
-
-
-@click.command()
-@click.option("--show-disabled", "-a", is_flag=True, help="Show hooks from disabled plugins too")
-@click.option("--model", "-m", type=str, default=None, help="Filter to specific model (Crawl, Snapshot, Binary, CrawlEnd)")
-@click.option("--quiet", "-q", is_flag=True, help="Output JSON only, no ASCII diagrams")
-@docstring(pluginmap.__doc__)
-def main(**kwargs):
-    import json
-
-    result = pluginmap(**kwargs)
-    if kwargs.get("quiet"):
-        print(json.dumps(result, indent=2))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,110 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox process <action> [--filters]
-
-Manage Process records (system-managed, mostly read-only).
-
-Process records track executions of binaries during extraction.
-They are created automatically by the system and are primarily for debugging.
-
-Actions:
-    list    - List Processes as JSONL (with optional filters)
-
-Examples:
-    # List all processes
-    archivebox process list
-
-    # List processes by binary
-    archivebox process list --binary-name=chrome
-
-    # List recent processes
-    archivebox process list --limit=10
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox process"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_utils import apply_filters
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_processes(
-    binary_name: str | None = None,
-    machine_id: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Processes as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Process
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Process.objects.all().select_related("binary", "machine").order_by("-start_ts")
-
-    # Apply filters
-    filter_kwargs = {}
-    if binary_name:
-        filter_kwargs["binary__name"] = binary_name
-    if machine_id:
-        filter_kwargs["machine_id"] = machine_id
-
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for process in queryset:
-        if is_tty:
-            binary_name_str = process.binary.name if process.binary else "unknown"
-            exit_code = process.exit_code if process.exit_code is not None else "?"
-            status_color = "green" if process.exit_code == 0 else "red" if process.exit_code else "yellow"
-            rprint(f"[{status_color}]exit={exit_code:3}[/{status_color}] [cyan]{binary_name_str:15}[/cyan] [dim]{process.id}[/dim]")
-        else:
-            write_record(process.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} processes[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Process records (read-only, system-managed)."""
-    pass
-
-
-@main.command("list")
-@click.option("--binary-name", "-b", help="Filter by binary name")
-@click.option("--machine-id", "-m", help="Filter by machine ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(binary_name: str | None, machine_id: str | None, limit: int | None):
-    """List Processes as JSONL."""
-    sys.exit(
-        list_processes(
-            binary_name=binary_name,
-            machine_id=machine_id,
-            limit=limit,
-        ),
-    )
-
-
-if __name__ == "__main__":
-    main()
@@ -1,112 +1,82 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
-__command__ = "archivebox remove"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox remove'

-import shutil
-from pathlib import Path
-from collections.abc import Iterable
+import sys
+import argparse

-import rich_click as click
+from typing import Optional, List, IO

-from django.db.models import QuerySet
-
-from archivebox.config import DATA_DIR
-from archivebox.config.constants import CONSTANTS
-from archivebox.config.django import setup_django
-from archivebox.misc.util import enforce_types, docstring
-from archivebox.misc.checks import check_data_folder
-from archivebox.misc.logging_util import (
-    log_list_started,
-    log_list_finished,
-    log_removal_started,
-    log_removal_finished,
-    TimedProgress,
-)
+from ..main import remove
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, accept_stdin


-@enforce_types
-def remove(
-    filter_patterns: Iterable[str] = (),
-    filter_type: str = "exact",
-    snapshots: QuerySet | None = None,
-    after: float | None = None,
-    before: float | None = None,
-    yes: bool = False,
-    delete: bool = False,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet:
-    """Remove the specified URLs from the archive"""
-
-    setup_django()
-    check_data_folder()
-
-    from archivebox.cli.archivebox_search import get_snapshots
-
-    pattern_list = list(filter_patterns)
-
-    log_list_started(pattern_list or None, filter_type)
-    timer = TimedProgress(360, prefix="      ")
-    try:
-        snapshots = get_snapshots(
-            snapshots=snapshots,
-            filter_patterns=pattern_list or None,
-            filter_type=filter_type,
-            after=after,
-            before=before,
-        )
-    finally:
-        timer.end()
-
-    if not snapshots.exists():
-        log_removal_finished(0, 0)
-        raise SystemExit(1)
-
-    log_list_finished(snapshots)
-    log_removal_started(snapshots, yes=yes, delete=delete)
-
-    timer = TimedProgress(360, prefix="      ")
-    try:
-        for snapshot in snapshots:
-            if delete:
-                shutil.rmtree(snapshot.output_dir, ignore_errors=True)
-                legacy_path = CONSTANTS.ARCHIVE_DIR / snapshot.timestamp
-                if legacy_path.is_symlink():
-                    legacy_path.unlink(missing_ok=True)
-    finally:
-        timer.end()
-
-    to_remove = snapshots.count()
-
-    from archivebox.search import flush_search_index
-    from archivebox.core.models import Snapshot
-
-    flush_search_index(snapshots=snapshots)
-    snapshots.delete()
-    all_snapshots = Snapshot.objects.all()
-    log_removal_finished(all_snapshots.count(), to_remove)
-
-    return all_snapshots
-
-
-@click.command()
-@click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--delete", is_flag=True, help="Delete the archived content and metadata folder in addition to removing from index")
-@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
-@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(("exact", "substring", "domain", "regex", "tag")),
-    default="exact",
-    help="Type of pattern matching to use when filtering URLs",
-)
-@click.argument("filter_patterns", nargs=-1)
@docstring(remove.__doc__)
-def main(**kwargs):
-    """Remove the specified URLs from the archive"""
-    remove(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=remove.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--yes', # '-y',
+        action='store_true',
+        help='Remove links instantly without prompting to confirm.',
+    )
+    parser.add_argument(
+        '--delete', # '-r',
+        action='store_true',
+        help=(
+            "In addition to removing the link from the index, "
+            "also delete its archived content and metadata folder."
+        ),
+    )
+    parser.add_argument(
+        '--before', #'-b',
+        type=float,
+        help="List only URLs bookmarked before the given timestamp.",
+        default=None,
+    )
+    parser.add_argument(
+        '--after', #'-a',
+        type=float,
+        help="List only URLs bookmarked after the given timestamp.",
+        default=None,
+    )
+    parser.add_argument(
+        '--filter-type',
+        type=str,
+        choices=('exact', 'substring', 'domain', 'regex','tag'),
+        default='exact',
+        help='Type of pattern matching to use when filtering URLs',
+    )
+    parser.add_argument(
+        'filter_patterns',
+        nargs='*',
+        type=str,
+        help='URLs matching this filter pattern will be removed from the index.'
+    )
+    command = parser.parse_args(args or ())
+    
+    filter_str = None
+    if not command.filter_patterns:
+        filter_str = accept_stdin(stdin)

+    remove(
+        filter_str=filter_str,
+        filter_patterns=command.filter_patterns,
+        filter_type=command.filter_type,
+        before=command.before,
+        after=command.after,
+        yes=command.yes,
+        delete=command.delete,
+        out_dir=pwd or OUTPUT_DIR,
+    )
+    

-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,359 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox run [--daemon] [--crawl-id=...] [--snapshot-id=...] [--binary-id=...]
-
-Unified command for processing queued work on the shared abx-dl bus.
-
-Modes:
-    - With stdin JSONL: Process piped records, exit when complete
-    - Without stdin (TTY): Run the background runner in foreground until killed
-    - --crawl-id: Run the crawl runner for a specific crawl only
-    - --snapshot-id: Run a specific snapshot through its parent crawl
-    - --binary-id: Emit a BinaryEvent for a specific Binary row
-
-Examples:
-    # Run the background runner in foreground
-    archivebox run
-
-    # Run as daemon (don't exit on idle)
-    archivebox run --daemon
-
-    # Process specific records (pipe any JSONL type, exits when done)
-    archivebox snapshot list --status=queued | archivebox run
-    archivebox archiveresult list --status=failed | archivebox run
-    archivebox crawl list --status=queued | archivebox run
-
-    # Mixed types work too
-    cat mixed_records.jsonl | archivebox run
-
-    # Run the crawl runner for a specific crawl
-    archivebox run --crawl-id=019b7e90-04d0-73ed-adec-aad9cfcd863e
-
-    # Run one snapshot from an existing crawl
-    archivebox run --snapshot-id=019b7e90-5a8e-712c-9877-2c70eebe80ad
-
-    # Run one queued binary install directly on the bus
-    archivebox run --binary-id=019b7e90-5a8e-712c-9877-2c70eebe80ad
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox run"
-
-import sys
-from collections import defaultdict
-
-import rich_click as click
-from rich import print as rprint
-
-
-def process_stdin_records() -> int:
-    """
-    Process JSONL records from stdin.
-
-    Create-or-update behavior:
-    - Records WITHOUT id: Create via Model.from_json(), then queue
-    - Records WITH id: Lookup existing, re-queue for processing
-
-    Outputs JSONL of all processed records (for chaining).
-
-    Handles any record type: Crawl, Snapshot, ArchiveResult.
-    Auto-cascades: Crawl → Snapshots → ArchiveResults.
-
-    Returns exit code (0 = success, 1 = error).
-    """
-    from django.utils import timezone
-
-    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_CRAWL, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT, TYPE_BINARY
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.core.models import Snapshot, ArchiveResult
-    from archivebox.crawls.models import Crawl
-    from archivebox.machine.models import Binary
-    from archivebox.services.runner import run_binary, run_crawl
-
-    records = list(read_stdin())
-    is_tty = sys.stdout.isatty()
-
-    if not records:
-        return 0  # Nothing to process
-
-    created_by_id = get_or_create_system_user_pk()
-    queued_count = 0
-    output_records = []
-    full_crawl_ids: set[str] = set()
-    snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
-    plugin_names_by_crawl: dict[str, set[str]] = defaultdict(set)
-    run_all_plugins_for_crawl: set[str] = set()
-    binary_ids: list[str] = []
-
-    for record in records:
-        record_type = record.get("type", "")
-        record_id = record.get("id")
-
-        try:
-            if record_type == TYPE_CRAWL:
-                if record_id:
-                    # Existing crawl - re-queue
-                    try:
-                        crawl = Crawl.objects.get(id=record_id)
-                    except Crawl.DoesNotExist:
-                        crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
-                else:
-                    # New crawl - create it
-                    crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
-
-                if crawl:
-                    crawl.retry_at = timezone.now()
-                    if crawl.status not in [Crawl.StatusChoices.SEALED]:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save()
-                    full_crawl_ids.add(str(crawl.id))
-                    run_all_plugins_for_crawl.add(str(crawl.id))
-                    output_records.append(crawl.to_json())
-                    queued_count += 1
-
-            elif record_type == TYPE_SNAPSHOT or (record.get("url") and not record_type):
-                if record_id:
-                    # Existing snapshot - re-queue
-                    try:
-                        snapshot = Snapshot.objects.get(id=record_id)
-                    except Snapshot.DoesNotExist:
-                        snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
-                else:
-                    # New snapshot - create it
-                    snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
-
-                if snapshot:
-                    snapshot.retry_at = timezone.now()
-                    if snapshot.status not in [Snapshot.StatusChoices.SEALED]:
-                        snapshot.status = Snapshot.StatusChoices.QUEUED
-                    snapshot.save()
-                    crawl = snapshot.crawl
-                    crawl.retry_at = timezone.now()
-                    if crawl.status != Crawl.StatusChoices.STARTED:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save(update_fields=["status", "retry_at", "modified_at"])
-                    crawl_id = str(snapshot.crawl_id)
-                    snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
-                    run_all_plugins_for_crawl.add(crawl_id)
-                    output_records.append(snapshot.to_json())
-                    queued_count += 1
-
-            elif record_type == TYPE_ARCHIVERESULT:
-                if record_id:
-                    # Existing archiveresult - re-queue
-                    try:
-                        archiveresult = ArchiveResult.objects.get(id=record_id)
-                    except ArchiveResult.DoesNotExist:
-                        archiveresult = None
-                else:
-                    archiveresult = None
-
-                snapshot_id = record.get("snapshot_id")
-                plugin_name = record.get("plugin")
-                snapshot = None
-                if archiveresult:
-                    if archiveresult.status in [
-                        ArchiveResult.StatusChoices.FAILED,
-                        ArchiveResult.StatusChoices.SKIPPED,
-                        ArchiveResult.StatusChoices.NORESULTS,
-                        ArchiveResult.StatusChoices.BACKOFF,
-                    ]:
-                        archiveresult.reset_for_retry()
-                    snapshot = archiveresult.snapshot
-                    plugin_name = plugin_name or archiveresult.plugin
-                elif snapshot_id:
-                    try:
-                        snapshot = Snapshot.objects.get(id=snapshot_id)
-                    except Snapshot.DoesNotExist:
-                        snapshot = None
-
-                if snapshot:
-                    snapshot.retry_at = timezone.now()
-                    if snapshot.status != Snapshot.StatusChoices.STARTED:
-                        snapshot.status = Snapshot.StatusChoices.QUEUED
-                    snapshot.save(update_fields=["status", "retry_at", "modified_at"])
-                    crawl = snapshot.crawl
-                    crawl.retry_at = timezone.now()
-                    if crawl.status != Crawl.StatusChoices.STARTED:
-                        crawl.status = Crawl.StatusChoices.QUEUED
-                    crawl.save(update_fields=["status", "retry_at", "modified_at"])
-                    crawl_id = str(snapshot.crawl_id)
-                    snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
-                    if plugin_name:
-                        plugin_names_by_crawl[crawl_id].add(str(plugin_name))
-                    output_records.append(record if not archiveresult else archiveresult.to_json())
-                    queued_count += 1
-
-            elif record_type == TYPE_BINARY:
-                if record_id:
-                    try:
-                        binary = Binary.objects.get(id=record_id)
-                    except Binary.DoesNotExist:
-                        binary = Binary.from_json(record)
-                else:
-                    binary = Binary.from_json(record)
-
-                if binary:
-                    binary.retry_at = timezone.now()
-                    if binary.status != Binary.StatusChoices.INSTALLED:
-                        binary.status = Binary.StatusChoices.QUEUED
-                    binary.save()
-                    binary_ids.append(str(binary.id))
-                    output_records.append(binary.to_json())
-                    queued_count += 1
-
-            else:
-                # Unknown type - pass through
-                output_records.append(record)
-
-        except Exception as e:
-            rprint(f"[yellow]Error processing record: {e}[/yellow]", file=sys.stderr)
-            continue
-
-    # Output all processed records (for chaining)
-    if not is_tty:
-        for rec in output_records:
-            write_record(rec)
-
-    if queued_count == 0:
-        rprint("[yellow]No records to process[/yellow]", file=sys.stderr)
-        return 0
-
-    rprint(f"[blue]Processing {queued_count} records...[/blue]", file=sys.stderr)
-
-    for binary_id in binary_ids:
-        run_binary(binary_id)
-
-    targeted_crawl_ids = full_crawl_ids | set(snapshot_ids_by_crawl)
-    if targeted_crawl_ids:
-        for crawl_id in sorted(targeted_crawl_ids):
-            run_crawl(
-                crawl_id,
-                snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
-                selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
-            )
-    return 0
-
-
-def run_runner(daemon: bool = False) -> int:
-    """
-    Run the background runner loop.
-
-    Args:
-        daemon: Run forever (don't exit when idle)
-
-    Returns exit code (0 = success, 1 = error).
-    """
-    from django.utils import timezone
-    from archivebox.machine.models import Machine, Process
-    from archivebox.services.runner import recover_orphaned_crawls, recover_orphaned_snapshots, run_pending_crawls
-
-    Process.cleanup_stale_running()
-    Process.cleanup_orphaned_workers()
-    recover_orphaned_snapshots()
-    recover_orphaned_crawls()
-    Machine.current()
-    current = Process.current()
-    if current.process_type != Process.TypeChoices.ORCHESTRATOR:
-        current.process_type = Process.TypeChoices.ORCHESTRATOR
-        current.save(update_fields=["process_type", "modified_at"])
-
-    try:
-        run_pending_crawls(daemon=daemon)
-        return 0
-    except KeyboardInterrupt:
-        return 0
-    except Exception as e:
-        rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-        return 1
-    finally:
-        current.refresh_from_db()
-        if current.status != Process.StatusChoices.EXITED:
-            current.status = Process.StatusChoices.EXITED
-            current.ended_at = current.ended_at or timezone.now()
-            current.save(update_fields=["status", "ended_at", "modified_at"])
-
-
-@click.command()
-@click.option("--daemon", "-d", is_flag=True, help="Run forever (don't exit on idle)")
-@click.option("--crawl-id", help="Run the crawl runner for a specific crawl only")
-@click.option("--snapshot-id", help="Run one snapshot through its crawl")
-@click.option("--binary-id", help="Run one queued binary install directly on the bus")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str):
-    """
-    Process queued work.
-
-    Modes:
-    - No args + stdin piped: Process piped JSONL records
-    - No args + TTY: Run the crawl runner for all work
-    - --crawl-id: Run the crawl runner for that crawl only
-    - --snapshot-id: Run one snapshot through its crawl only
-    - --binary-id: Run one queued binary install directly on the bus
-    """
-    if snapshot_id:
-        sys.exit(run_snapshot_worker(snapshot_id))
-
-    if binary_id:
-        try:
-            from archivebox.services.runner import run_binary
-
-            run_binary(binary_id)
-            sys.exit(0)
-        except KeyboardInterrupt:
-            sys.exit(0)
-        except Exception as e:
-            rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-            import traceback
-
-            traceback.print_exc()
-            sys.exit(1)
-
-    if crawl_id:
-        try:
-            from archivebox.services.runner import run_crawl
-
-            run_crawl(crawl_id)
-            sys.exit(0)
-        except KeyboardInterrupt:
-            sys.exit(0)
-        except Exception as e:
-            rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-            import traceback
-
-            traceback.print_exc()
-            sys.exit(1)
-
-    if daemon:
-        if not sys.stdin.isatty():
-            exit_code = process_stdin_records()
-            if exit_code != 0:
-                sys.exit(exit_code)
-        sys.exit(run_runner(daemon=True))
-
-    if not sys.stdin.isatty():
-        sys.exit(process_stdin_records())
-    else:
-        sys.exit(run_runner(daemon=daemon))
-
-
-def run_snapshot_worker(snapshot_id: str) -> int:
-    from archivebox.core.models import Snapshot
-    from archivebox.services.runner import run_crawl
-
-    try:
-        snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)])
-        return 0
-    except KeyboardInterrupt:
-        return 0
-    except Exception as e:
-        rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-        import traceback
-
-        traceback.print_exc()
-        return 1
-
-
-if __name__ == "__main__":
-    main()
@@ -1,177 +1,116 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox schedule'

-import rich_click as click
-from rich import print
+import sys
+import argparse

-from archivebox.misc.util import enforce_types, docstring
-from archivebox.config.common import ARCHIVING_CONFIG
+from typing import Optional, List, IO
+
+from ..main import schedule
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin


-@enforce_types
-def schedule(
-    add: bool = False,
-    show: bool = False,
-    clear: bool = False,
-    foreground: bool = False,
-    run_all: bool = False,
-    quiet: bool = False,
-    every: str | None = None,
-    tag: str = "",
-    depth: int | str = 0,
-    overwrite: bool = False,
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW,
-    import_path: str | None = None,
-):
-    """Manage database-backed scheduled crawls processed by the crawl runner."""
-
-    from django.utils import timezone
-
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.crawls.models import Crawl, CrawlSchedule
-    from archivebox.crawls.schedule_utils import validate_schedule
-    from archivebox.services.runner import run_pending_crawls
-
-    depth = int(depth)
-    result: dict[str, object] = {
-        "created_schedule_ids": [],
-        "disabled_count": 0,
-        "run_all_enqueued": 0,
-        "active_schedule_ids": [],
-    }
-
-    def _active_schedules():
-        return CrawlSchedule.objects.filter(is_enabled=True).select_related("template").order_by("created_at")
-
-    if clear:
-        disabled_count = CrawlSchedule.objects.filter(is_enabled=True).update(
-            is_enabled=False,
-            modified_at=timezone.now(),
-        )
-        result["disabled_count"] = disabled_count
-        print(f"[green]\\[√] Disabled {disabled_count} scheduled crawl(s).[/green]")
-
-    if every or add:
-        schedule_str = (every or "day").strip()
-        validate_schedule(schedule_str)
-
-        created_by_id = get_or_create_system_user_pk()
-        is_update_schedule = not import_path
-        template_urls = import_path or "archivebox://update"
-        template_label = (f"Scheduled import: {template_urls}" if import_path else "Scheduled ArchiveBox update")[:64]
-        template_notes = (
-            f"Created by archivebox schedule for {template_urls}"
-            if import_path
-            else "Created by archivebox schedule to queue recurring archivebox://update maintenance crawls."
-        )
-
-        template = Crawl.objects.create(
-            urls=template_urls,
-            max_depth=0 if is_update_schedule else depth,
-            tags_str="" if is_update_schedule else tag,
-            label=template_label,
-            notes=template_notes,
-            created_by_id=created_by_id,
-            status=Crawl.StatusChoices.SEALED,
-            retry_at=None,
-            config={
-                "ONLY_NEW": not update,
-                "OVERWRITE": overwrite,
-                "DEPTH": 0 if is_update_schedule else depth,
-                "SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
-            },
-        )
-        crawl_schedule = CrawlSchedule.objects.create(
-            template=template,
-            schedule=schedule_str,
-            is_enabled=True,
-            label=template_label,
-            notes=template_notes,
-            created_by_id=created_by_id,
-        )
-        result["created_schedule_ids"] = [str(crawl_schedule.id)]
-
-        schedule_type = "maintenance update" if is_update_schedule else "crawl"
-        print(f"[green]\\[√] Created scheduled {schedule_type}.[/green]")
-        print(f"    id={crawl_schedule.id}")
-        print(f"    every={crawl_schedule.schedule}")
-        print(f"    next_run={crawl_schedule.next_run_at.isoformat()}")
-        if import_path:
-            print(f"    source={import_path}")
-
-    schedules = list(_active_schedules())
-    result["active_schedule_ids"] = [str(schedule.id) for schedule in schedules]
-
-    if show:
-        if schedules:
-            print(f"[green]\\[*] Active scheduled crawls: {len(schedules)}[/green]")
-            for scheduled_crawl in schedules:
-                template = scheduled_crawl.template
-                print(
-                    f"  - id={scheduled_crawl.id} every={scheduled_crawl.schedule} "
-                    f"next_run={scheduled_crawl.next_run_at.isoformat()} "
-                    f"source={template.urls.splitlines()[0] if template.urls else ''}",
-                )
-        else:
-            print("[yellow]\\[*] No scheduled crawls are enabled.[/yellow]")
-
-    if run_all:
-        enqueued = 0
-        now = timezone.now()
-        for scheduled_crawl in schedules:
-            scheduled_crawl.enqueue(queued_at=now)
-            enqueued += 1
-        result["run_all_enqueued"] = enqueued
-        print(f"[green]\\[*] Enqueued {enqueued} scheduled crawl(s) immediately.[/green]")
-        if enqueued:
-            print(
-                "[yellow]\\[*] Start `archivebox server`, `archivebox run --daemon`, or `archivebox schedule --foreground` to process the queued crawls.[/yellow]",
-            )
-
-    if foreground:
-        print(
-            "[green]\\[*] Starting global crawl runner in foreground mode. It will materialize scheduled crawls and process queued work.[/green]",
-        )
-        run_pending_crawls(daemon=True)
-
-    if quiet:
-        return result
-
-    if not any((every, add, show, clear, foreground, run_all)):
-        if schedules:
-            print("[green]\\[*] Active scheduled crawls:[/green]")
-            for scheduled_crawl in schedules:
-                print(f"  - {scheduled_crawl.id} every={scheduled_crawl.schedule} next_run={scheduled_crawl.next_run_at.isoformat()}")
-        else:
-            print("[yellow]\\[*] No scheduled crawls are enabled.[/yellow]")
-
-    return result
-
-
-@click.command()
-@click.option("--quiet", "-q", is_flag=True, help="Return structured results without extra summary output")
-@click.option("--add", is_flag=True, help="Create a new scheduled crawl")
-@click.option("--every", type=str, help='Run on an alias like daily/weekly/monthly or a cron expression such as "0 */6 * * *"')
-@click.option("--tag", "-t", default="", help="Comma-separated tags to apply to scheduled crawl snapshots")
-@click.option(
-    "--depth",
-    type=click.Choice([str(i) for i in range(5)]),
-    default="0",
-    help="Recursively archive linked pages up to N hops away",
-)
-@click.option("--overwrite", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
-@click.option("--update", is_flag=True, help="Retry previously failed/skipped URLs when scheduled crawls run")
-@click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
-@click.option("--show", is_flag=True, help="Print all currently enabled schedules")
-@click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
-@click.option("--run-all", is_flag=True, help="Enqueue all enabled schedules immediately and process them once")
-@click.argument("import_path", required=False)
@docstring(schedule.__doc__)
-def main(**kwargs):
-    """Manage database-backed scheduled crawls processed by the crawl runner."""
-    schedule(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=schedule.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--quiet', '-q',
+        action='store_true',
+        help=("Don't warn about storage space."),
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--add', # '-a',
+        action='store_true',
+        help='Add a new scheduled ArchiveBox update job to cron',
+    )
+    parser.add_argument(
+        '--every', # '-e',
+        type=str,
+        default=None,
+        help='Run ArchiveBox once every [timeperiod] (hour/day/month/year or cron format e.g. "0 0 * * *")',
+    )
+    parser.add_argument(
+        '--tag', '-t',
+        type=str,
+        default='',
+        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
+    )
+    parser.add_argument(
+        '--depth', # '-d',
+        type=int,
+        choices=[0, 1],
+        default=0,
+        help='Depth to archive to [0] or 1, see "add" command help for more info',
+    )
+    parser.add_argument(
+        '--overwrite',
+        action='store_true',
+        help='Re-archive any URLs that have been previously archived, overwriting existing Snapshots',
+    )
+    parser.add_argument(
+        '--update',
+        action='store_true',
+        help='Re-pull any URLs that have been previously added, as needed to fill missing ArchiveResults',
+    )
+    group.add_argument(
+        '--clear', # '-c'
+        action='store_true',
+        help=("Stop all ArchiveBox scheduled runs (remove cron jobs)"),
+    )
+    group.add_argument(
+        '--show', # '-s'
+        action='store_true',
+        help=("Print a list of currently active ArchiveBox cron jobs"),
+    )
+    group.add_argument(
+        '--foreground', '-f',
+        action='store_true',
+        help=("Launch ArchiveBox scheduler as a long-running foreground task "
+              "instead of using cron."),
+    )
+    group.add_argument(
+        '--run-all', # '-a',
+        action='store_true',
+        help=("Run all the scheduled jobs once immediately, independent of "
+              "their configured schedules, can be used together with --foreground"),
+    )
+    parser.add_argument(
+        'import_path',
+        nargs='?',
+        type=str,
+        default=None,
+        help=("Check this path and import any new links on every run "
+              "(can be either local file or remote URL)"),
+    )
+    command = parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)
+
+    schedule(
+        add=command.add,
+        show=command.show,
+        clear=command.clear,
+        foreground=command.foreground,
+        run_all=command.run_all,
+        quiet=command.quiet,
+        every=command.every,
+        tag=command.tag,
+        depth=command.depth,
+        overwrite=command.overwrite,
+        update=command.update,
+        import_path=command.import_path,
+        out_dir=pwd or OUTPUT_DIR,
+    )


-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,257 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox search"
-
-import sys
-from pathlib import Path
-from typing import TYPE_CHECKING
-from collections.abc import Callable
-
-import rich_click as click
-
-from django.db.models import Q, QuerySet
-
-from archivebox.config import DATA_DIR
-from archivebox.misc.logging import stderr
-from archivebox.misc.util import enforce_types, docstring
-
-if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
-
-# Filter types for URL matching
-LINK_FILTERS: dict[str, Callable[[str], Q]] = {
-    "exact": lambda pattern: Q(url=pattern),
-    "substring": lambda pattern: Q(url__icontains=pattern),
-    "regex": lambda pattern: Q(url__iregex=pattern),
-    "domain": lambda pattern: (
-        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
-    ),
-    "tag": lambda pattern: Q(tags__name=pattern),
-    "timestamp": lambda pattern: Q(timestamp=pattern),
-}
-
-STATUS_CHOICES = ["indexed", "archived", "unarchived"]
-
-
-def _apply_pattern_filters(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    filter_patterns: list[str],
-    filter_type: str,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    filter_builder = LINK_FILTERS.get(filter_type)
-    if filter_builder is None:
-        stderr()
-        stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red")
-        raise SystemExit(2)
-
-    query = Q()
-    for pattern in filter_patterns:
-        query |= filter_builder(pattern)
-    return snapshots.filter(query)
-
-
-def _snapshots_to_json(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from archivebox.config import VERSION
-    from archivebox.config.common import SERVER_CONFIG
-    from archivebox.misc.util import to_json
-
-    main_index_header = (
-        {
-            "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
-            "schema": "archivebox.index.json",
-            "copyright_info": SERVER_CONFIG.FOOTER_INFO,
-            "meta": {
-                "project": "ArchiveBox",
-                "version": VERSION,
-                "git_sha": VERSION,
-                "website": "https://ArchiveBox.io",
-                "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
-                "source": "https://github.com/ArchiveBox/ArchiveBox",
-                "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
-                "dependencies": {},
-            },
-        }
-        if with_headers
-        else {}
-    )
-
-    snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)]
-    output: dict[str, object] | list[dict[str, object]]
-    if with_headers:
-        output = {
-            **main_index_header,
-            "num_links": len(snapshot_dicts),
-            "updated": datetime.now(tz.utc),
-            "last_run_cmd": sys.argv,
-            "links": snapshot_dicts,
-        }
-    else:
-        output = snapshot_dicts
-
-    return to_json(output, indent=4, sort_keys=True)
-
-
-def _snapshots_to_csv(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    cols: list[str],
-    with_headers: bool,
-) -> str:
-    header = ",".join(cols) if with_headers else ""
-    rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)]
-    return "\n".join((header, *rows))
-
-
-def _snapshots_to_html(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from django.template.loader import render_to_string
-
-    from archivebox.config import VERSION
-    from archivebox.config.common import SERVER_CONFIG
-    from archivebox.config.version import get_COMMIT_HASH
-
-    template = "static_index.html" if with_headers else "minimal_index.html"
-    snapshot_list = list(snapshots.iterator(chunk_size=500))
-
-    return render_to_string(
-        template,
-        {
-            "version": VERSION,
-            "git_sha": get_COMMIT_HASH() or VERSION,
-            "num_links": str(len(snapshot_list)),
-            "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
-            "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
-            "links": snapshot_list,
-            "FOOTER_INFO": SERVER_CONFIG.FOOTER_INFO,
-        },
-    )
-
-
-def get_snapshots(
-    snapshots: QuerySet["Snapshot", "Snapshot"] | None = None,
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    after: float | None = None,
-    before: float | None = None,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    """Filter and return Snapshots matching the given criteria."""
-    from archivebox.core.models import Snapshot
-
-    if snapshots is not None:
-        result = snapshots
-    else:
-        result = Snapshot.objects.all()
-
-    if after is not None:
-        result = result.filter(timestamp__gte=after)
-    if before is not None:
-        result = result.filter(timestamp__lt=before)
-    if filter_patterns:
-        result = _apply_pattern_filters(result, filter_patterns, filter_type)
-
-    # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir
-    result = result.select_related("crawl", "crawl__created_by")
-
-    if not result.exists():
-        stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow")
-
-    return result
-
-
-@enforce_types
-def search(
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    status: str = "indexed",
-    before: float | None = None,
-    after: float | None = None,
-    sort: str | None = None,
-    json: bool = False,
-    html: bool = False,
-    csv: str | None = None,
-    with_headers: bool = False,
-):
-    """List, filter, and export information about archive entries"""
-
-    if with_headers and not (json or html or csv):
-        stderr("[X] --with-headers requires --json, --html or --csv\n", color="red")
-        raise SystemExit(2)
-
-    # Query DB directly - no filesystem scanning
-    snapshots = get_snapshots(
-        filter_patterns=list(filter_patterns) if filter_patterns else None,
-        filter_type=filter_type,
-        before=before,
-        after=after,
-    )
-
-    # Apply status filter
-    if status == "archived":
-        snapshots = snapshots.filter(downloaded_at__isnull=False)
-    elif status == "unarchived":
-        snapshots = snapshots.filter(downloaded_at__isnull=True)
-    # 'indexed' = all snapshots (no filter)
-
-    if sort:
-        snapshots = snapshots.order_by(sort)
-
-    # Export to requested format
-    if json:
-        output = _snapshots_to_json(snapshots, with_headers=with_headers)
-    elif html:
-        output = _snapshots_to_html(snapshots, with_headers=with_headers)
-    elif csv:
-        output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers)
-    else:
-        from archivebox.misc.logging_util import printable_folders
-
-        # Convert to dict for printable_folders
-        folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots}
-        output = printable_folders(folders, with_headers)
-
-    # Structured exports must be written directly to stdout.
-    # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output.
-    sys.stdout.write(output)
-    if not output.endswith("\n"):
-        sys.stdout.write("\n")
-    return output
-
-
-@click.command()
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(["search", *LINK_FILTERS.keys()]),
-    default="substring",
-    help="Pattern matching type for filtering URLs",
-)
-@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status")
-@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp")
-@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--json", "-J", is_flag=True, help="Print output in JSON format")
-@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title")
-@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output")
-@click.help_option("--help", "-h")
-@click.argument("filter_patterns", nargs=-1)
-@docstring(search.__doc__)
-def main(**kwargs):
-    return search(**kwargs)
-
-
-if __name__ == "__main__":
-    main()
@@ -1,235 +1,76 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox server'

-from collections.abc import Iterable
 import sys
+import argparse

-import rich_click as click
-from rich import print
+from typing import Optional, List, IO

-from archivebox.misc.util import docstring, enforce_types
-from archivebox.config.common import SERVER_CONFIG
+from ..main import server
+from ..util import docstring
+from ..config import OUTPUT_DIR, BIND_ADDR
+from ..logging_util import SmartFormatter, reject_stdin

-
-def stop_existing_background_runner(*, machine, process_model, supervisor=None, stop_worker_fn=None, log=print) -> int:
-    """Stop any existing orchestrator process so the server can take ownership."""
-    process_model.cleanup_stale_running(machine=machine)
-    process_model.cleanup_orphaned_workers()
-
-    running_runners = list(
-        process_model.objects.filter(
-            machine=machine,
-            status=process_model.StatusChoices.RUNNING,
-            process_type=process_model.TypeChoices.ORCHESTRATOR,
-        ).order_by("created_at"),
-    )
-
-    if not running_runners:
-        return 0
-
-    log("[yellow][*] Stopping existing ArchiveBox background runner...[/yellow]")
-
-    if supervisor is not None and stop_worker_fn is not None:
-        for worker_name in ("worker_runner", "worker_runner_watch"):
-            try:
-                stop_worker_fn(supervisor, worker_name)
-            except Exception:
-                pass
-
-    for proc in running_runners:
-        try:
-            proc.kill_tree(graceful_timeout=2.0)
-        except Exception:
-            try:
-                proc.terminate(graceful_timeout=2.0)
-            except Exception:
-                pass
-
-    process_model.cleanup_stale_running(machine=machine)
-    return len(running_runners)
-
-
-def _read_supervisor_worker_command(worker_name: str) -> str:
-    from archivebox.workers.supervisord_util import WORKERS_DIR_NAME, get_sock_file
-
-    worker_conf = get_sock_file().parent / WORKERS_DIR_NAME / f"{worker_name}.conf"
-    if not worker_conf.exists():
-        return ""
-
-    for line in worker_conf.read_text().splitlines():
-        if line.startswith("command="):
-            return line.removeprefix("command=").strip()
-    return ""
-
-
-def _worker_command_matches_bind(command: str, host: str, port: str) -> bool:
-    if not command:
-        return False
-    return f"{host}:{port}" in command or (f"--bind={host}" in command and f"--port={port}" in command)
-
-
-def stop_existing_server_workers(*, supervisor, stop_worker_fn, host: str, port: str, log=print) -> int:
-    """Stop existing ArchiveBox web workers if they already own the requested bind."""
-    stopped = 0
-
-    for worker_name in ("worker_runserver", "worker_daphne"):
-        try:
-            proc = supervisor.getProcessInfo(worker_name) if supervisor else None
-        except Exception:
-            proc = None
-        if not isinstance(proc, dict) or proc.get("statename") != "RUNNING":
-            continue
-
-        command = _read_supervisor_worker_command(worker_name)
-        if not _worker_command_matches_bind(command, host, port):
-            continue
-
-        if stopped == 0:
-            log("[yellow][*] Taking over existing ArchiveBox web server on same port...[/yellow]")
-        stop_worker_fn(supervisor, worker_name)
-        stopped += 1
-
-    return stopped
-
-
-@enforce_types
-def server(
-    runserver_args: Iterable[str] = (SERVER_CONFIG.BIND_ADDR,),
-    reload: bool = False,
-    init: bool = False,
-    debug: bool = False,
-    daemonize: bool = False,
-    nothreading: bool = False,
-) -> None:
-    """Run the ArchiveBox HTTP server"""
-
-    runserver_args = list(runserver_args)
-
-    if init:
-        from archivebox.cli.archivebox_init import init as archivebox_init
-
-        archivebox_init(quick=True)
-        print()
-
-    from archivebox.misc.checks import check_data_folder
-
-    check_data_folder()
-
-    from archivebox.config.common import SHELL_CONFIG
-
-    run_in_debug = SHELL_CONFIG.DEBUG or debug or reload
-    if debug or reload:
-        SHELL_CONFIG.DEBUG = True
-
-    from django.contrib.auth.models import User
-
-    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
-        print()
-        print(
-            "[violet]Hint:[/violet] To create an [bold]admin username & password[/bold] for the [deep_sky_blue3][underline][link=http://{host}:{port}/admin]Admin UI[/link][/underline][/deep_sky_blue3], run:",
-        )
-        print("      [green]archivebox manage createsuperuser[/green]")
-        print()
-
-    host = "127.0.0.1"
-    port = "8000"
-
-    try:
-        host_and_port = [arg for arg in runserver_args if arg.replace(".", "").replace(":", "").isdigit()][0]
-        if ":" in host_and_port:
-            host, port = host_and_port.split(":")
-        else:
-            if "." in host_and_port:
-                host = host_and_port
-            else:
-                port = host_and_port
-    except IndexError:
-        pass
-
-    from archivebox.workers.supervisord_util import (
-        get_existing_supervisord_process,
-        get_worker,
-        stop_worker,
-        start_server_workers,
-        is_port_in_use,
-    )
-    from archivebox.machine.models import Machine, Process
-
-    machine = Machine.current()
-    supervisor = get_existing_supervisord_process()
-    stop_existing_background_runner(
-        machine=machine,
-        process_model=Process,
-        supervisor=supervisor,
-        stop_worker_fn=stop_worker,
-    )
-    if supervisor:
-        stop_existing_server_workers(
-            supervisor=supervisor,
-            stop_worker_fn=stop_worker,
-            host=host,
-            port=port,
-        )
-
-    # Check if port is already in use
-    if is_port_in_use(host, int(port)):
-        print(f"[red][X] Error: Port {port} is already in use[/red]")
-        print(f"    Another process (possibly daphne or runserver) is already listening on {host}:{port}")
-        print("    Stop the conflicting process or choose a different port")
-        sys.exit(1)
-
-    supervisor = get_existing_supervisord_process()
-    if supervisor:
-        server_worker_name = "worker_runserver" if run_in_debug else "worker_daphne"
-        server_proc = get_worker(supervisor, server_worker_name)
-        server_state = server_proc.get("statename") if isinstance(server_proc, dict) else None
-        if server_state == "RUNNING":
-            runner_proc = get_worker(supervisor, "worker_runner")
-            runner_watch_proc = get_worker(supervisor, "worker_runner_watch")
-            runner_state = runner_proc.get("statename") if isinstance(runner_proc, dict) else None
-            runner_watch_state = runner_watch_proc.get("statename") if isinstance(runner_watch_proc, dict) else None
-            print("[red][X] Error: ArchiveBox server is already running[/red]")
-            print(
-                f"    [green]√[/green] Web server ({server_worker_name}) is RUNNING on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
-            )
-            if runner_state == "RUNNING":
-                print("    [green]√[/green] Background runner (worker_runner) is RUNNING")
-            if runner_watch_state == "RUNNING":
-                print("    [green]√[/green] Reload watcher (worker_runner_watch) is RUNNING")
-            print()
-            print("[yellow]To stop the existing server, run:[/yellow]")
-            print('    pkill -f "archivebox server"')
-            print("    pkill -f supervisord")
-            sys.exit(1)
-
-    if run_in_debug:
-        print("[green][+] Starting ArchiveBox webserver in DEBUG mode...[/green]")
-    else:
-        print("[green][+] Starting ArchiveBox webserver...[/green]")
-    print(
-        f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
-    )
-    print(
-        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link=http://{host}:{port}/admin]http://{host}:{port}/admin[/link][/deep_sky_blue3]",
-    )
-    print("    > Writing ArchiveBox error log to ./logs/errors.log")
-    print()
-    start_server_workers(host=host, port=port, daemonize=daemonize, debug=run_in_debug, reload=reload, nothreading=nothreading)
-    print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
-
-
-@click.command()
-@click.argument("runserver_args", nargs=-1)
-@click.option("--reload", is_flag=True, help="Enable auto-reloading when code or templates change")
-@click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
-@click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
-@click.option("--init", is_flag=True, help="Run a full archivebox init/upgrade before starting the server")
-@click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
@docstring(server.__doc__)
-def main(**kwargs):
-    server(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=server.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        'runserver_args',
+        nargs='*',
+        type=str,
+        default=[BIND_ADDR],
+        help='Arguments to pass to Django runserver'
+    )
+    parser.add_argument(
+        '--reload',
+        action='store_true',
+        help='Enable auto-reloading when code or templates change',
+    )
+    parser.add_argument(
+        '--debug',
+        action='store_true',
+        help='Enable DEBUG=True mode with more verbose errors',
+    )
+    parser.add_argument(
+        '--nothreading',
+        action='store_true',
+        help='Force runserver to run in single-threaded mode',
+    )
+    parser.add_argument(
+        '--init',
+        action='store_true',
+        help='Run a full archivebox init/upgrade before starting the server',
+    )
+    parser.add_argument(
+        '--quick-init', '-i',
+        action='store_true',
+        help='Run quick archivebox init/upgrade before starting the server',
+    )
+    parser.add_argument(
+        '--createsuperuser',
+        action='store_true',
+        help='Run archivebox manage createsuperuser before starting the server',
+    )
+    command = parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)
+    
+    server(
+        runserver_args=command.runserver_args + (['--nothreading'] if command.nothreading else []),
+        reload=command.reload,
+        debug=command.debug,
+        init=command.init,
+        quick_init=command.quick_init,
+        createsuperuser=command.createsuperuser,
+        out_dir=pwd or OUTPUT_DIR,
+    )


-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox setup'
+
+import sys
+import argparse
+
+from typing import Optional, List, IO
+
+from ..main import setup
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin
+
+
+@docstring(setup.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=setup.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    # parser.add_argument(
+    #     '--force', # '-f',
+    #     action='store_true',
+    #     help='Overwrite any existing packages that conflict with the ones ArchiveBox is trying to install',
+    # )
+    command = parser.parse_args(args or ())   # noqa
+    reject_stdin(__command__, stdin)
+
+    setup(
+        # force=command.force,
+        out_dir=pwd or OUTPUT_DIR,
+    )
+    
+
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,28 +1,34 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox shell'

-from collections.abc import Iterable
+import sys
+import argparse

-import rich_click as click
+from typing import Optional, List, IO

-from archivebox.misc.util import docstring
+from ..main import shell
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin


-def shell(args: Iterable[str] = ()) -> None:
-    """Enter an interactive ArchiveBox Django shell"""
-
-    from django.core.management import call_command
-
-    call_command("shell_plus", *args)
-
-
-@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
-@click.argument("args", nargs=-1)
@docstring(shell.__doc__)
-def main(args: Iterable[str] = ()) -> None:
-    shell(args=args)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=shell.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)
+    
+    shell(
+        out_dir=pwd or OUTPUT_DIR,
+    )
+    

-
-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,497 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox snapshot <action> [args...] [--filters]
-
-Manage Snapshot records.
-
-Actions:
-    create  - Create Snapshots from URLs or Crawl JSONL
-    list    - List Snapshots as JSONL (with optional filters)
-    update  - Update Snapshots from stdin JSONL
-    delete  - Delete Snapshots from stdin JSONL
-
-Examples:
-    # Create
-    archivebox snapshot create https://example.com --tag=news
-    archivebox crawl create https://example.com | archivebox snapshot create
-
-    # List with filters
-    archivebox snapshot list --status=queued
-    archivebox snapshot list --url__icontains=example.com
-
-    # Update
-    archivebox snapshot list --tag=old | archivebox snapshot update --tag=new
-
-    # Delete
-    archivebox snapshot list --url__icontains=spam.com | archivebox snapshot delete --yes
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox snapshot"
-
-import sys
-from collections.abc import Iterable
-
-import rich_click as click
-from rich import print as rprint
-from django.db.models import Q, Sum
-from django.db.models.functions import Coalesce
-
-from archivebox.cli.cli_utils import apply_filters
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_snapshots(
-    urls: Iterable[str],
-    tag: str = "",
-    status: str = "queued",
-    depth: int = 0,
-    created_by_id: int | None = None,
-) -> int:
-    """
-    Create Snapshots from URLs or stdin JSONL (Crawl or Snapshot records).
-    Pass-through: Records that are not Crawl/Snapshot/URL are output unchanged.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import (
-        read_args_or_stdin,
-        write_record,
-        TYPE_SNAPSHOT,
-        TYPE_CRAWL,
-    )
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.core.models import Snapshot
-    from archivebox.crawls.models import Crawl
-
-    created_by_id = created_by_id or get_or_create_system_user_pk()
-    is_tty = sys.stdout.isatty()
-
-    # Collect all input records
-    records = list(read_args_or_stdin(urls))
-
-    if not records:
-        rprint("[yellow]No URLs or Crawls provided. Pass URLs as arguments or via stdin.[/yellow]", file=sys.stderr)
-        return 1
-
-    # Process each record - handle Crawls and plain URLs/Snapshots
-    created_snapshots = []
-    pass_through_count = 0
-
-    for record in records:
-        record_type = record.get("type", "")
-
-        try:
-            if record_type == TYPE_CRAWL:
-                # Pass through the Crawl record itself first
-                if not is_tty:
-                    write_record(record)
-
-                # Input is a Crawl - get or create it, then create Snapshots for its URLs
-                crawl = None
-                crawl_id = record.get("id")
-                if crawl_id:
-                    try:
-                        crawl = Crawl.objects.get(id=crawl_id)
-                    except Crawl.DoesNotExist:
-                        crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
-                else:
-                    crawl = Crawl.from_json(record, overrides={"created_by_id": created_by_id})
-
-                if not crawl:
-                    continue
-
-                # Create snapshots for each URL in the crawl
-                for url in crawl.get_urls_list():
-                    merged_tags = crawl.tags_str
-                    if tag:
-                        merged_tags = f"{merged_tags},{tag}" if merged_tags else tag
-                    snapshot_record = {
-                        "url": url,
-                        "tags": merged_tags,
-                        "crawl_id": str(crawl.id),
-                        "depth": depth,
-                        "status": status,
-                    }
-                    snapshot = Snapshot.from_json(snapshot_record, overrides={"created_by_id": created_by_id})
-                    if snapshot:
-                        created_snapshots.append(snapshot)
-                        if not is_tty:
-                            write_record(snapshot.to_json())
-
-            elif record_type == TYPE_SNAPSHOT or record.get("url"):
-                # Input is a Snapshot or plain URL
-                if tag and not record.get("tags"):
-                    record["tags"] = tag
-                if status:
-                    record["status"] = status
-                record["depth"] = record.get("depth", depth)
-
-                snapshot = Snapshot.from_json(record, overrides={"created_by_id": created_by_id})
-                if snapshot:
-                    created_snapshots.append(snapshot)
-                    if not is_tty:
-                        write_record(snapshot.to_json())
-
-            else:
-                # Pass-through: output records we don't handle
-                if not is_tty:
-                    write_record(record)
-                pass_through_count += 1
-
-        except Exception as e:
-            rprint(f"[red]Error creating snapshot: {e}[/red]", file=sys.stderr)
-            continue
-
-    if not created_snapshots:
-        if pass_through_count > 0:
-            rprint(f"[dim]Passed through {pass_through_count} records, no new snapshots[/dim]", file=sys.stderr)
-            return 0
-        rprint("[red]No snapshots created[/red]", file=sys.stderr)
-        return 1
-
-    rprint(f"[green]Created {len(created_snapshots)} snapshots[/green]", file=sys.stderr)
-
-    if is_tty:
-        for snapshot in created_snapshots:
-            rprint(f"  [dim]{snapshot.id}[/dim] {snapshot.url[:60]}", file=sys.stderr)
-
-    return 0
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_snapshots(
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
-    csv: str | None = None,
-    with_headers: bool = False,
-    search: str | None = None,
-    query: str | None = None,
-) -> int:
-    """
-    List Snapshots as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import Snapshot
-    from archivebox.search import (
-        get_default_search_mode,
-        get_search_mode,
-        prioritize_metadata_matches,
-        query_search_index,
-    )
-
-    if with_headers and not csv:
-        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
-        return 2
-
-    is_tty = sys.stdout.isatty() and not csv
-
-    queryset = Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)).order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "url__icontains": url__icontains,
-        "url__istartswith": url__istartswith,
-        "crawl_id": crawl_id,
-    }
-    queryset = apply_filters(queryset, filter_kwargs)
-
-    # Tag filter requires special handling (M2M)
-    if tag:
-        queryset = queryset.filter(tags__name__iexact=tag)
-
-    query = (query or "").strip()
-    if query:
-        metadata_qs = queryset.filter(
-            Q(title__icontains=query) | Q(url__icontains=query) | Q(timestamp__icontains=query) | Q(tags__name__icontains=query),
-        )
-        requested_search_mode = (search or "").strip().lower()
-        if requested_search_mode == "content":
-            requested_search_mode = "contents"
-        search_mode = get_default_search_mode() if not requested_search_mode else get_search_mode(requested_search_mode)
-
-        if search_mode == "meta":
-            queryset = metadata_qs
-        else:
-            try:
-                deep_qsearch = None
-                if search_mode == "deep":
-                    qsearch = query_search_index(query, search_mode="contents")
-                    deep_qsearch = query_search_index(query, search_mode="deep")
-                else:
-                    qsearch = query_search_index(query, search_mode=search_mode)
-                queryset = prioritize_metadata_matches(
-                    queryset,
-                    metadata_qs,
-                    qsearch,
-                    deep_queryset=deep_qsearch,
-                    ordering=("-created_at",) if not sort else None,
-                )
-            except Exception as err:
-                rprint(
-                    f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
-                    file=sys.stderr,
-                )
-                queryset = metadata_qs
-
-    if sort:
-        queryset = queryset.order_by(sort)
-    if limit:
-        queryset = queryset[:limit]
-
-    count = 0
-    if csv:
-        cols = [col.strip() for col in csv.split(",") if col.strip()]
-        if not cols:
-            rprint("[red]No CSV columns provided[/red]", file=sys.stderr)
-            return 2
-        rows: list[str] = []
-        if with_headers:
-            rows.append(",".join(cols))
-        for snapshot in queryset.iterator(chunk_size=500):
-            rows.append(snapshot.to_csv(cols=cols, separator=","))
-            count += 1
-        output = "\n".join(rows)
-        if output:
-            sys.stdout.write(output)
-            if not output.endswith("\n"):
-                sys.stdout.write("\n")
-        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
-        return 0
-
-    for snapshot in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(snapshot.status, "dim")
-            rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
-        else:
-            write_record(snapshot.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_snapshots(
-    status: str | None = None,
-    tag: str | None = None,
-) -> int:
-    """
-    Update Snapshots from stdin JSONL.
-
-    Reads Snapshot records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from django.utils import timezone
-
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.core.models import Snapshot
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        snapshot_id = record.get("id")
-        if not snapshot_id:
-            continue
-
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-
-            # Apply updates from CLI flags (override stdin values)
-            if status:
-                snapshot.status = status
-                snapshot.retry_at = timezone.now()
-            if tag:
-                # Add tag to existing tags
-                snapshot.save()  # Ensure saved before M2M
-                from archivebox.core.models import Tag
-
-                tag_obj, _ = Tag.objects.get_or_create(name=tag)
-                snapshot.tags.add(tag_obj)
-
-            snapshot.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(snapshot.to_json())
-
-        except Snapshot.DoesNotExist:
-            rprint(f"[yellow]Snapshot not found: {snapshot_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} snapshots[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_snapshots(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Snapshots from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.core.models import Snapshot
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    snapshot_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not snapshot_ids:
-        rprint("[yellow]No valid snapshot IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    snapshots = Snapshot.objects.filter(id__in=snapshot_ids)
-    count = snapshots.count()
-
-    if count == 0:
-        rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} snapshots (dry run)[/yellow]", file=sys.stderr)
-        for snapshot in snapshots:
-            rprint(f"  [dim]{snapshot.id}[/dim] {snapshot.url[:60]}", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = snapshots.delete()
-    rprint(f"[green]Deleted {deleted_count} snapshots[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Snapshot records."""
-    pass
-
-
-@main.command("create")
-@click.argument("urls", nargs=-1)
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
-def create_cmd(urls: tuple, tag: str, status: str, depth: int):
-    """Create Snapshots from URLs or stdin JSONL."""
-    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
-
-
-@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def list_cmd(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-):
-    """List Snapshots as JSONL."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--status", "-s", help="Set status")
-@click.option("--tag", "-t", help="Add tag")
-def update_cmd(status: str | None, tag: str | None):
-    """Update Snapshots from stdin JSONL."""
-    sys.exit(update_snapshots(status=status, tag=tag))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Snapshots from stdin JSONL."""
-    sys.exit(delete_snapshots(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox snapshot"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_snapshot import create_snapshots
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
-@click.argument("urls", nargs=-1)
-def main(tag: str, status: str, depth: int, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox snapshot URL...` entrypoint."""
-    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,139 +1,32 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox status'

-from pathlib import Path
+import sys
+import argparse

-import rich_click as click
-from rich import print
+from typing import Optional, List, IO

-from archivebox.misc.util import enforce_types, docstring
-from archivebox.config import DATA_DIR, CONSTANTS, ARCHIVE_DIR
-from archivebox.config.common import SHELL_CONFIG
-from archivebox.misc.legacy import parse_json_links_details
-from archivebox.misc.system import get_dir_size
-from archivebox.misc.logging_util import printable_filesize
+from ..main import status
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin


-@enforce_types
-def status(out_dir: Path = DATA_DIR) -> None:
-    """Print out some info and statistics about the archive collection"""
-
-    from django.contrib.auth import get_user_model
-    from django.db.models import Sum
-    from django.db.models.functions import Coalesce
-    from archivebox.core.models import Snapshot
-
-    User = get_user_model()
-
-    print("[green]\\[*] Scanning archive main index...[/green]")
-    print(f"[yellow]   {out_dir}/*[/yellow]")
-    num_bytes, num_dirs, num_files = get_dir_size(out_dir, recursive=False, pattern="index.")
-    size = printable_filesize(num_bytes)
-    print(f"    Index size: {size} across {num_files} files")
-    print()
-
-    links = list(Snapshot.objects.annotate(output_size_sum=Coalesce(Sum("archiveresult__output_size"), 0)))
-    num_sql_links = len(links)
-    num_link_details = sum(1 for link in parse_json_links_details(out_dir=out_dir))
-    print(f"    > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {CONSTANTS.SQL_INDEX_FILENAME})")
-    print(f"    > JSON Link Details: {num_link_details} links".ljust(36), f"(found in {ARCHIVE_DIR.name}/*/index.json)")
-    print()
-    print("[green]\\[*] Scanning archive data directories...[/green]")
-    users_dir = out_dir / "users"
-    scan_roots = [root for root in (ARCHIVE_DIR, users_dir) if root.exists()]
-    scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(ARCHIVE_DIR)
-    print(f"[yellow]   {scan_roots_display}[/yellow]")
-    num_bytes = num_dirs = num_files = 0
-    for root in scan_roots:
-        root_bytes, root_dirs, root_files = get_dir_size(root)
-        num_bytes += root_bytes
-        num_dirs += root_dirs
-        num_files += root_files
-    size = printable_filesize(num_bytes)
-    print(f"    Size: {size} across {num_files} files in {num_dirs} directories")
-
-    # Use DB as source of truth for snapshot status
-    num_indexed = len(links)
-    num_archived = sum(1 for snapshot in links if snapshot.is_archived)
-    num_unarchived = max(num_indexed - num_archived, 0)
-    print(f"    > indexed: {num_indexed}".ljust(36), "(total snapshots in DB)")
-    print(f"      > archived: {num_archived}".ljust(36), "(snapshots with archived content)")
-    print(f"      > unarchived: {num_unarchived}".ljust(36), "(snapshots pending archiving)")
-
-    # Count snapshot directories on filesystem across both legacy and current layouts.
-    expected_snapshot_dirs = {str(Path(snapshot.output_dir).resolve()) for snapshot in links if Path(snapshot.output_dir).exists()}
-    discovered_snapshot_dirs = set()
-
-    if ARCHIVE_DIR.exists():
-        discovered_snapshot_dirs.update(str(entry.resolve()) for entry in ARCHIVE_DIR.iterdir() if entry.is_dir())
-
-    if users_dir.exists():
-        discovered_snapshot_dirs.update(str(entry.resolve()) for entry in users_dir.glob("*/snapshots/*/*/*") if entry.is_dir())
-
-    orphaned_dirs = sorted(discovered_snapshot_dirs - expected_snapshot_dirs)
-    num_present = len(discovered_snapshot_dirs)
-    num_valid = len(discovered_snapshot_dirs & expected_snapshot_dirs)
-    print()
-    print(f"    > present: {num_present}".ljust(36), "(snapshot directories on disk)")
-    print(f"      > [green]valid:[/green] {num_valid}".ljust(36), "               (directories with matching DB entry)")
-
-    num_orphaned = len(orphaned_dirs)
-    print(f"      > [red]orphaned:[/red] {num_orphaned}".ljust(36), "         (directories without matching DB entry)")
-
-    if num_indexed:
-        print("    [violet]Hint:[/violet] You can list snapshots by status like so:")
-        print("        [green]archivebox list --status=<status>  (e.g. archived, queued, etc.)[/green]")
-
-    if orphaned_dirs:
-        print("    [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")
-        print("        [green]archivebox init[/green]")
-
-    print()
-    print("[green]\\[*] Scanning recent archive changes and user logins:[/green]")
-    print(f"[yellow]   {CONSTANTS.LOGS_DIR}/*[/yellow]")
-    admin_users = User.objects.filter(is_superuser=True).exclude(username="system")
-    users = [user.get_username() for user in admin_users]
-    print(f"    UI users {len(users)}: {', '.join(users)}")
-    last_login = admin_users.order_by("last_login").last()
-    if last_login:
-        print(f"    Last UI login: {last_login.get_username()} @ {str(last_login.last_login)[:16]}")
-    last_downloaded = Snapshot.objects.order_by("downloaded_at").last()
-    if last_downloaded:
-        print(f"    Last changes: {str(last_downloaded.downloaded_at)[:16]}")
-
-    if not users:
-        print()
-        print("    [violet]Hint:[/violet] You can create an admin user by running:")
-        print("        [green]archivebox manage createsuperuser[/green]")
-
-    print()
-    recent_snapshots = sorted(
-        links,
-        key=lambda snapshot: snapshot.downloaded_at or snapshot.modified_at or snapshot.created_at,
-        reverse=True,
-    )[:10]
-    for snapshot in recent_snapshots:
-        if not snapshot.downloaded_at:
-            continue
-        print(
-            (
-                "[grey53] "
-                f"   > {str(snapshot.downloaded_at)[:16]} "
-                f"[{snapshot.num_outputs} {('X', '√')[snapshot.is_archived]} {printable_filesize(snapshot.archive_size)}] "
-                f'"{snapshot.title}": {snapshot.url}'
-                "[/grey53]"
-            )[: SHELL_CONFIG.TERM_WIDTH],
-        )
-    print("[grey53]   ...")
-
-
-@click.command()
@docstring(status.__doc__)
-def main(**kwargs):
-    """Print out some info and statistics about the archive collection"""
-    status(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=status.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)
+
+    status(out_dir=pwd or OUTPUT_DIR)


-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,299 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox tag <action> [args...] [--filters]
-
-Manage Tag records.
-
-Actions:
-    create  - Create Tags
-    list    - List Tags as JSONL (with optional filters)
-    update  - Update Tags from stdin JSONL
-    delete  - Delete Tags from stdin JSONL
-
-Examples:
-    # Create
-    archivebox tag create news tech science
-    archivebox tag create "important stuff"
-
-    # List
-    archivebox tag list
-    archivebox tag list --name__icontains=news
-
-    # Update (rename tags)
-    archivebox tag list --name=oldname | archivebox tag update --name=newname
-
-    # Delete
-    archivebox tag list --name=unused | archivebox tag delete --yes
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox tag"
-
-import sys
-from collections.abc import Iterable
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_utils import apply_filters
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_tags(names: Iterable[str]) -> int:
-    """
-    Create Tags from names.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import Tag
-
-    is_tty = sys.stdout.isatty()
-
-    # Convert to list if needed
-    name_list = list(names) if names else []
-
-    if not name_list:
-        rprint("[yellow]No tag names provided. Pass names as arguments.[/yellow]", file=sys.stderr)
-        return 1
-
-    created_count = 0
-    for name in name_list:
-        name = name.strip()
-        if not name:
-            continue
-
-        tag, created = Tag.objects.get_or_create(name=name)
-
-        if not is_tty:
-            write_record(tag.to_json())
-
-        if created:
-            created_count += 1
-            rprint(f"[green]Created tag: {name}[/green]", file=sys.stderr)
-        else:
-            rprint(f"[dim]Tag already exists: {name}[/dim]", file=sys.stderr)
-
-    rprint(f"[green]Created {created_count} new tags[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_tags(
-    name: str | None = None,
-    name__icontains: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Tags as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import Tag
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Tag.objects.all().order_by("name")
-
-    # Apply filters
-    filter_kwargs = {
-        "name": name,
-        "name__icontains": name__icontains,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for tag in queryset:
-        snapshot_count = tag.snapshot_set.count()
-        if is_tty:
-            rprint(f"[cyan]{tag.name:30}[/cyan] [dim]({snapshot_count} snapshots)[/dim]")
-        else:
-            write_record(tag.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} tags[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_tags(name: str | None = None) -> int:
-    """
-    Update Tags from stdin JSONL.
-
-    Reads Tag records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.core.models import Tag
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        tag_id = record.get("id")
-        old_name = record.get("name")
-
-        if not tag_id and not old_name:
-            continue
-
-        try:
-            if tag_id:
-                tag = Tag.objects.get(id=tag_id)
-            else:
-                tag = Tag.objects.get(name=old_name)
-
-            # Apply updates from CLI flags
-            if name:
-                tag.name = name
-                tag.save()
-
-            updated_count += 1
-
-            if not is_tty:
-                write_record(tag.to_json())
-
-        except Tag.DoesNotExist:
-            rprint(f"[yellow]Tag not found: {tag_id or old_name}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} tags[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_tags(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Tags from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.core.models import Tag
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    # Collect tag IDs or names
-    tag_ids = []
-    tag_names = []
-    for r in records:
-        if r.get("id"):
-            tag_ids.append(r["id"])
-        elif r.get("name"):
-            tag_names.append(r["name"])
-
-    if not tag_ids and not tag_names:
-        rprint("[yellow]No valid tag IDs or names in input[/yellow]", file=sys.stderr)
-        return 1
-
-    from django.db.models import Q
-
-    query = Q()
-    if tag_ids:
-        query |= Q(id__in=tag_ids)
-    if tag_names:
-        query |= Q(name__in=tag_names)
-
-    tags = Tag.objects.filter(query)
-    count = tags.count()
-
-    if count == 0:
-        rprint("[yellow]No matching tags found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} tags (dry run)[/yellow]", file=sys.stderr)
-        for tag in tags:
-            rprint(f"  {tag.name}", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = tags.delete()
-    rprint(f"[green]Deleted {deleted_count} tags[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Tag records."""
-    pass
-
-
-@main.command("create")
-@click.argument("names", nargs=-1)
-def create_cmd(names: tuple):
-    """Create Tags from names."""
-    sys.exit(create_tags(names))
-
-
-@main.command("list")
-@click.option("--name", help="Filter by exact name")
-@click.option("--name__icontains", help="Filter by name contains")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
-    """List Tags as JSONL."""
-    sys.exit(list_tags(name=name, name__icontains=name__icontains, limit=limit))
-
-
-@main.command("update")
-@click.option("--name", "-n", help="Set new name")
-def update_cmd(name: str | None):
-    """Update Tags from stdin JSONL."""
-    sys.exit(update_tags(name=name))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Tags from stdin JSONL."""
-    sys.exit(delete_tags(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,600 +1,136 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox update'
+
+import sys
+import argparse
+
+from typing import List, Optional, IO
+
+from ..main import update
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..index import (
+    LINK_FILTERS,
+    get_indexed_folders,
+    get_archived_folders,
+    get_unarchived_folders,
+    get_present_folders,
+    get_valid_folders,
+    get_invalid_folders,
+    get_duplicate_folders,
+    get_orphaned_folders,
+    get_corrupted_folders,
+    get_unrecognized_folders,
+)
+from ..logging_util import SmartFormatter, accept_stdin

-import os
-import time

-from typing import TYPE_CHECKING, Any
-from collections.abc import Callable, Iterable
-from pathlib import Path
-
-import rich_click as click
-from django.core.exceptions import ObjectDoesNotExist
-from django.db.models import Q, QuerySet
-
-from archivebox.misc.util import enforce_types, docstring
-
-if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
-    from archivebox.crawls.models import Crawl
-
-
-LINK_FILTERS: dict[str, Callable[[str], Q]] = {
-    "exact": lambda pattern: Q(url=pattern),
-    "substring": lambda pattern: Q(url__icontains=pattern),
-    "regex": lambda pattern: Q(url__iregex=pattern),
-    "domain": lambda pattern: (
-        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
-    ),
-    "tag": lambda pattern: Q(tags__name=pattern),
-    "timestamp": lambda pattern: Q(timestamp=pattern),
-}
-
-
-def _apply_pattern_filters(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    filter_patterns: list[str],
-    filter_type: str,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    filter_builder = LINK_FILTERS.get(filter_type)
-    if filter_builder is None:
-        raise SystemExit(2)
-
-    query = Q()
-    for pattern in filter_patterns:
-        query |= filter_builder(pattern)
-    return snapshots.filter(query)
-
-
-def _get_snapshot_crawl(snapshot: "Snapshot") -> "Crawl | None":
-    try:
-        return snapshot.crawl
-    except ObjectDoesNotExist:
-        return None
-
-
-def _get_search_indexing_plugins() -> list[str]:
-    from abx_dl.models import discover_plugins
-    from archivebox.hooks import get_search_backends
-
-    available_backends = set(get_search_backends())
-    plugins = discover_plugins()
-    return sorted(
-        plugin_name
-        for plugin_name, plugin in plugins.items()
-        if plugin_name.startswith("search_backend_")
-        and plugin_name.removeprefix("search_backend_") in available_backends
-        and any("Snapshot" in hook.name and "index" in hook.name.lower() for hook in plugin.hooks)
-    )
-
-
-def _build_filtered_snapshots_queryset(
-    *,
-    filter_patterns: Iterable[str],
-    filter_type: str,
-    before: float | None,
-    after: float | None,
-    resume: str | None = None,
-):
-    from archivebox.core.models import Snapshot
-    from datetime import datetime
-
-    snapshots = Snapshot.objects.all()
-
-    if filter_patterns:
-        snapshots = _apply_pattern_filters(snapshots, list(filter_patterns), filter_type)
-
-    if before:
-        snapshots = snapshots.filter(bookmarked_at__lt=datetime.fromtimestamp(before))
-    if after:
-        snapshots = snapshots.filter(bookmarked_at__gt=datetime.fromtimestamp(after))
-    if resume:
-        snapshots = snapshots.filter(timestamp__lte=resume)
-
-    return snapshots.select_related("crawl").order_by("-bookmarked_at")
-
-
-def reindex_snapshots(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    search_plugins: list[str],
-    batch_size: int,
-) -> dict[str, int]:
-    from archivebox.cli.archivebox_extract import run_plugins
-
-    stats = {"processed": 0, "reconciled": 0, "queued": 0, "reindexed": 0}
-    records: list[dict[str, str]] = []
-
-    total = snapshots.count()
-    print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
-
-    for snapshot in snapshots.iterator(chunk_size=batch_size):
-        stats["processed"] += 1
-
-        if _get_snapshot_crawl(snapshot) is None:
-            continue
-
-        output_dir = Path(snapshot.output_dir)
-        has_directory = output_dir.exists() and output_dir.is_dir()
-        if has_directory:
-            snapshot.reconcile_with_index_json()
-            stats["reconciled"] += 1
-
-        for plugin_name in search_plugins:
-            existing_result = snapshot.archiveresult_set.filter(plugin=plugin_name).order_by("-created_at").first()
-            if existing_result:
-                existing_result.reset_for_retry()
-            records.append(
-                {
-                    "type": "ArchiveResult",
-                    "snapshot_id": str(snapshot.id),
-                    "plugin": plugin_name,
-                },
-            )
-            stats["queued"] += 1
-
-    if not records:
-        return stats
-
-    exit_code = run_plugins(
-        args=(),
-        records=records,
-        wait=True,
-        emit_results=False,
-    )
-    if exit_code != 0:
-        raise SystemExit(exit_code)
-
-    stats["reindexed"] = len(records)
-    return stats
-
-
-@enforce_types
-def update(
-    filter_patterns: Iterable[str] = (),
-    filter_type: str = "exact",
-    before: float | None = None,
-    after: float | None = None,
-    resume: str | None = None,
-    batch_size: int = 100,
-    continuous: bool = False,
-    index_only: bool = False,
-) -> None:
-    """
-    Update snapshots: migrate old dirs, reconcile DB, and re-queue for archiving.
-
-    Three-phase operation (without filters):
-    - Phase 1: Drain old archive/ dirs by moving to new fs location (0.8.x → 0.9.x)
-    - Phase 2: O(n) scan over entire DB from most recent to least recent
-    - No orphan scans needed (trust 1:1 mapping between DB and filesystem after phase 1)
-
-    With filters: Only phase 2 (DB query), no filesystem operations.
-    Without filters: All phases (full update).
-    """
-
-    from rich import print
-    from archivebox.config.django import setup_django
-
-    setup_django()
-
-    from django.core.management import call_command
-
-    # Run migrations first to ensure DB schema is up-to-date
-    print("[*] Checking for pending migrations...")
-    try:
-        call_command("migrate", "--no-input", verbosity=0)
-    except Exception as e:
-        print(f"[!] Warning: Migration check failed: {e}")
-
-    while True:
-        if index_only:
-            search_plugins = _get_search_indexing_plugins()
-            if not search_plugins:
-                print("[*] No search indexing plugins are available, nothing to backfill.")
-                break
-
-            if not (filter_patterns or before or after):
-                print("[*] Phase 1: Draining old archive/ directories (0.8.x → 0.9.x migration)...")
-                drain_old_archive_dirs(
-                    resume_from=resume,
-                    batch_size=batch_size,
-                )
-
-            snapshots = _build_filtered_snapshots_queryset(
-                filter_patterns=filter_patterns,
-                filter_type=filter_type,
-                before=before,
-                after=after,
-                resume=resume,
-            )
-            stats = reindex_snapshots(
-                snapshots,
-                search_plugins=search_plugins,
-                batch_size=batch_size,
-            )
-            print_index_stats(stats)
-        elif filter_patterns or before or after:
-            # Filtered mode: query DB only
-            print("[*] Processing filtered snapshots from database...")
-            stats = process_filtered_snapshots(
-                filter_patterns=filter_patterns,
-                filter_type=filter_type,
-                before=before,
-                after=after,
-                resume=resume,
-                batch_size=batch_size,
-            )
-            print_stats(stats)
-        else:
-            # Full mode: drain old dirs + process DB
-            stats_combined = {"phase1": {}, "phase2": {}}
-
-            print("[*] Phase 1: Draining old archive/ directories (0.8.x → 0.9.x migration)...")
-            stats_combined["phase1"] = drain_old_archive_dirs(
-                resume_from=resume,
-                batch_size=batch_size,
-            )
-
-            print("[*] Phase 2: Processing all database snapshots (most recent first)...")
-            stats_combined["phase2"] = process_all_db_snapshots(batch_size=batch_size, resume=resume)
-
-            # Phase 3: Deduplication (disabled for now)
-            # print('[*] Phase 3: Deduplicating...')
-            # stats_combined['deduplicated'] = Snapshot.find_and_merge_duplicates()
-
-            print_combined_stats(stats_combined)
-
-        if not continuous:
-            break
-
-        print("[yellow]Sleeping 60s before next pass...[/yellow]")
-        time.sleep(60)
-        resume = None
-
-
-def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 100) -> dict[str, int]:
-    """
-    Drain old archive/ directories (0.8.x → 0.9.x migration).
-
-    Only processes real directories (skips symlinks - those are already migrated).
-    For each old dir found in archive/:
-      1. Load or create DB snapshot
-      2. Trigger fs migration on save() to move to data/users/{user}/...
-      3. Leave symlink in archive/ pointing to new location
-
-    After this drains, archive/ should only contain symlinks and we can trust
-    1:1 mapping between DB and filesystem.
-    """
-    from archivebox.core.models import Snapshot
-    from archivebox.config import CONSTANTS
-    from django.db import transaction
-
-    stats = {"processed": 0, "migrated": 0, "skipped": 0, "invalid": 0}
-
-    archive_dir = CONSTANTS.ARCHIVE_DIR
-    if not archive_dir.exists():
-        return stats
-
-    print("[DEBUG Phase1] Scanning for old directories in archive/...")
-
-    # Scan for real directories only (skip symlinks - they're already migrated)
-    all_entries = list(os.scandir(archive_dir))
-    print(f"[DEBUG Phase1] Total entries in archive/: {len(all_entries)}")
-    entries = [
-        (e.stat().st_mtime, e.path)
-        for e in all_entries
-        if e.is_dir(follow_symlinks=False)  # Skip symlinks
-    ]
-    entries.sort(reverse=True)  # Newest first
-    print(f"[DEBUG Phase1] Real directories (not symlinks): {len(entries)}")
-    print(f"[*] Found {len(entries)} old directories to drain")
-
-    for mtime, entry_path in entries:
-        entry_path = Path(entry_path)
-
-        # Resume from timestamp if specified
-        if resume_from and entry_path.name > resume_from:
-            continue
-
-        stats["processed"] += 1
-
-        # Try to load existing snapshot from DB
-        snapshot = Snapshot.load_from_directory(entry_path)
-
-        if not snapshot:
-            # Not in DB - create new snapshot record
-            snapshot = Snapshot.create_from_directory(entry_path)
-            if not snapshot:
-                # Invalid directory - move to invalid/
-                Snapshot.move_directory_to_invalid(entry_path)
-                stats["invalid"] += 1
-                print(f"    [{stats['processed']}] Invalid: {entry_path.name}")
-                continue
-
-            try:
-                snapshot.save()
-                stats["migrated"] += 1
-                print(f"    [{stats['processed']}] Imported orphaned snapshot: {entry_path.name}")
-            except Exception as e:
-                stats["skipped"] += 1
-                print(f"    [{stats['processed']}] Skipped (error: {e}): {entry_path.name}")
-            continue
-
-        # Ensure snapshot has a valid crawl (migration 0024 may have failed)
-        has_valid_crawl = _get_snapshot_crawl(snapshot) is not None
-
-        if not has_valid_crawl:
-            # Create a new crawl (created_by will default to system user)
-            from archivebox.crawls.models import Crawl
-
-            crawl = Crawl.objects.create(urls=snapshot.url)
-            # Use queryset update to avoid triggering save() hooks
-            from archivebox.core.models import Snapshot as SnapshotModel
-
-            SnapshotModel.objects.filter(pk=snapshot.pk).update(crawl=crawl)
-            # Refresh the instance
-            snapshot.crawl = crawl
-            print(f"[DEBUG Phase1] Created missing crawl for snapshot {str(snapshot.id)[:8]}")
-
-        # Check if needs migration (0.8.x → 0.9.x)
-        print(
-            f"[DEBUG Phase1] Snapshot {str(snapshot.id)[:8]}: fs_version={snapshot.fs_version}, needs_migration={snapshot.fs_migration_needed}",
-        )
-        if snapshot.fs_migration_needed:
-            try:
-                # Calculate paths using actual directory (entry_path), not snapshot.timestamp
-                # because snapshot.timestamp might be truncated
-                old_dir = entry_path
-                new_dir = snapshot.get_storage_path_for_version("0.9.0")
-                print(f"[DEBUG Phase1] Migrating {old_dir.name} → {new_dir}")
-
-                # Manually migrate files
-                if not new_dir.exists() and old_dir.exists():
-                    new_dir.mkdir(parents=True, exist_ok=True)
-                    import shutil
-
-                    file_count = 0
-                    for old_file in old_dir.rglob("*"):
-                        if old_file.is_file():
-                            rel_path = old_file.relative_to(old_dir)
-                            new_file = new_dir / rel_path
-                            if not new_file.exists():
-                                new_file.parent.mkdir(parents=True, exist_ok=True)
-                                shutil.copy2(old_file, new_file)
-                                file_count += 1
-                    print(f"[DEBUG Phase1] Copied {file_count} files")
-
-                # Update only fs_version field using queryset update (bypasses validation)
-                from archivebox.core.models import Snapshot as SnapshotModel
-
-                SnapshotModel.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0")
-
-                # Commit the transaction
-                transaction.commit()
-
-                # Cleanup: delete old dir and create symlink
-                if old_dir.exists() and old_dir != new_dir:
-                    snapshot._cleanup_old_migration_dir(old_dir, new_dir)
-
-                stats["migrated"] += 1
-                print(f"    [{stats['processed']}] Migrated: {entry_path.name}")
-            except Exception as e:
-                stats["skipped"] += 1
-                print(f"    [{stats['processed']}] Skipped (error: {e}): {entry_path.name}")
-        else:
-            stats["skipped"] += 1
-
-        if stats["processed"] % batch_size == 0:
-            transaction.commit()
-
-    transaction.commit()
-    return stats
-
-
-def process_all_db_snapshots(batch_size: int = 100, resume: str | None = None) -> dict[str, int]:
-    """
-    O(n) scan over entire DB from most recent to least recent.
-
-    For each snapshot:
-      1. Reconcile index.json with DB (merge titles, tags, archive results)
-      2. Queue for archiving (state machine will handle it)
-
-    No orphan detection needed - we trust 1:1 mapping between DB and filesystem
-    after Phase 1 has drained all old archive/ directories.
-    """
-    from archivebox.core.models import Snapshot
-    from django.db import transaction
-    from django.utils import timezone
-
-    stats = {"processed": 0, "reconciled": 0, "queued": 0}
-
-    queryset = Snapshot.objects.all()
-    if resume:
-        queryset = queryset.filter(timestamp__lte=resume)
-    total = queryset.count()
-    print(f"[*] Processing {total} snapshots from database (most recent first)...")
-
-    # Process from most recent to least recent
-    for snapshot in queryset.select_related("crawl").order_by("-bookmarked_at").iterator(chunk_size=batch_size):
-        stats["processed"] += 1
-
-        # Skip snapshots with missing crawl references (orphaned by migration errors)
-        if _get_snapshot_crawl(snapshot) is None:
-            continue
-
-        try:
-            print(
-                f"[DEBUG Phase2] Snapshot {str(snapshot.id)[:8]}: fs_version={snapshot.fs_version}, needs_migration={snapshot.fs_migration_needed}",
-            )
-
-            # Check if snapshot has a directory on disk
-            from pathlib import Path
-
-            output_dir = Path(snapshot.output_dir)
-            has_directory = output_dir.exists() and output_dir.is_dir()
-
-            # Only reconcile if directory exists (don't create empty directories for orphans)
-            if has_directory:
-                snapshot.reconcile_with_index_json()
-
-            # Clean up invalid field values from old migrations
-            if not isinstance(snapshot.current_step, int):
-                snapshot.current_step = 0
-
-            # If still needs migration, it's an orphan (no directory on disk)
-            # Mark it as migrated to prevent save() from triggering filesystem migration
-            if snapshot.fs_migration_needed:
-                if has_directory:
-                    print(f"[DEBUG Phase2] WARNING: Snapshot {str(snapshot.id)[:8]} has directory but still needs migration")
-                else:
-                    print(f"[DEBUG Phase2] Orphan snapshot {str(snapshot.id)[:8]} - marking as migrated without filesystem operation")
-                # Use queryset update to set fs_version without triggering save() hooks
-                from archivebox.core.models import Snapshot as SnapshotModel
-
-                SnapshotModel.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0")
-                snapshot.fs_version = "0.9.0"
-
-            # Queue for archiving (state machine will handle it)
-            snapshot.status = Snapshot.StatusChoices.QUEUED
-            snapshot.retry_at = timezone.now()
-            snapshot.save()
-
-            stats["reconciled"] += 1 if has_directory else 0
-            stats["queued"] += 1
-        except Exception as e:
-            # Skip snapshots that can't be processed (e.g., missing crawl)
-            print(f"    [!] Skipping snapshot {snapshot.id}: {e}")
-            continue
-
-        if stats["processed"] % batch_size == 0:
-            transaction.commit()
-            print(f"    [{stats['processed']}/{total}] Processed...")
-
-    transaction.commit()
-    return stats
-
-
-def process_filtered_snapshots(
-    filter_patterns: Iterable[str],
-    filter_type: str,
-    before: float | None,
-    after: float | None,
-    resume: str | None,
-    batch_size: int,
-) -> dict[str, int]:
-    """Process snapshots matching filters (DB query only)."""
-    from django.db import transaction
-    from django.utils import timezone
-
-    stats = {"processed": 0, "reconciled": 0, "queued": 0}
-
-    snapshots = _build_filtered_snapshots_queryset(
-        filter_patterns=filter_patterns,
-        filter_type=filter_type,
-        before=before,
-        after=after,
-        resume=resume,
-    )
-
-    total = snapshots.count()
-    print(f"[*] Found {total} matching snapshots")
-
-    for snapshot in snapshots.select_related("crawl").iterator(chunk_size=batch_size):
-        stats["processed"] += 1
-
-        # Skip snapshots with missing crawl references
-        if _get_snapshot_crawl(snapshot) is None:
-            continue
-
-        try:
-            # Reconcile index.json with DB
-            snapshot.reconcile_with_index_json()
-
-            # Clean up invalid field values from old migrations
-            if not isinstance(snapshot.current_step, int):
-                snapshot.current_step = 0
-
-            # Queue for archiving
-            snapshot.status = Snapshot.StatusChoices.QUEUED
-            snapshot.retry_at = timezone.now()
-            snapshot.save()
-
-            stats["reconciled"] += 1
-            stats["queued"] += 1
-        except Exception as e:
-            # Skip snapshots that can't be processed
-            print(f"    [!] Skipping snapshot {snapshot.id}: {e}")
-            continue
-
-        if stats["processed"] % batch_size == 0:
-            transaction.commit()
-            print(f"    [{stats['processed']}/{total}] Processed...")
-
-    transaction.commit()
-    return stats
-
-
-def print_stats(stats: dict):
-    """Print statistics for filtered mode."""
-    from rich import print
-
-    print(f"""
-[green]Update Complete[/green]
-  Processed:   {stats["processed"]}
-  Reconciled:  {stats["reconciled"]}
-  Queued:      {stats["queued"]}
-""")
-
-
-def print_combined_stats(stats_combined: dict):
-    """Print statistics for full mode."""
-    from rich import print
-
-    s1 = stats_combined["phase1"]
-    s2 = stats_combined["phase2"]
-
-    print(f"""
-[green]Archive Update Complete[/green]
-
-Phase 1 (Drain Old Dirs):
-  Checked:     {s1.get("processed", 0)}
-  Migrated:    {s1.get("migrated", 0)}
-  Skipped:     {s1.get("skipped", 0)}
-  Invalid:     {s1.get("invalid", 0)}
-
-Phase 2 (Process DB):
-  Processed:   {s2.get("processed", 0)}
-  Reconciled:  {s2.get("reconciled", 0)}
-  Queued:      {s2.get("queued", 0)}
-""")
-
-
-def print_index_stats(stats: dict[str, Any]) -> None:
-    from rich import print
-
-    print(f"""
-[green]Search Reindex Complete[/green]
-  Processed:   {stats["processed"]}
-  Reconciled:  {stats["reconciled"]}
-  Queued:      {stats["queued"]}
-  Reindexed:   {stats["reindexed"]}
-""")
-
-
-@click.command()
-@click.option("--resume", type=str, help="Resume from timestamp")
-@click.option("--before", type=float, help="Only snapshots before timestamp")
-@click.option("--after", type=float, help="Only snapshots after timestamp")
-@click.option("--filter-type", "-t", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")
-@click.option("--batch-size", type=int, default=100, help="Commit every N snapshots")
-@click.option("--continuous", is_flag=True, help="Run continuously as background worker")
-@click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
-@click.argument("filter_patterns", nargs=-1)
@docstring(update.__doc__)
-def main(**kwargs):
-    update(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=update.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--only-new', #'-n',
+        action='store_true',
+        help="Don't attempt to retry previously skipped/failed links when updating",
+    )
+    parser.add_argument(
+        '--index-only', #'-o',
+        action='store_true',
+        help="Update the main index without archiving any content",
+    )
+    parser.add_argument(
+        '--resume', #'-r',
+        type=float,
+        help='Resume the update process from a given timestamp',
+        default=None,
+    )
+    parser.add_argument(
+        '--overwrite', #'-x',
+        action='store_true',
+        help='Ignore existing archived content and overwrite with new versions (DANGEROUS)',
+    )
+    parser.add_argument(
+        '--before', #'-b',
+        type=float,
+        help="Update only links bookmarked before the given timestamp.",
+        default=None,
+    )
+    parser.add_argument(
+        '--after', #'-a',
+        type=float,
+        help="Update only links bookmarked after the given timestamp.",
+        default=None,
+    )
+    parser.add_argument(
+        '--status',
+        type=str,
+        choices=('indexed', 'archived', 'unarchived', 'present', 'valid', 'invalid', 'duplicate', 'orphaned', 'corrupted', 'unrecognized'),
+        default='indexed',
+        help=(
+            'Update only links or data directories that have the given status\n'
+            f'    indexed       {get_indexed_folders.__doc__} (the default)\n'
+            f'    archived      {get_archived_folders.__doc__}\n'
+            f'    unarchived    {get_unarchived_folders.__doc__}\n'
+            '\n'
+            f'    present       {get_present_folders.__doc__}\n'
+            f'    valid         {get_valid_folders.__doc__}\n'
+            f'    invalid       {get_invalid_folders.__doc__}\n'
+            '\n'
+            f'    duplicate     {get_duplicate_folders.__doc__}\n'
+            f'    orphaned      {get_orphaned_folders.__doc__}\n'
+            f'    corrupted     {get_corrupted_folders.__doc__}\n'
+            f'    unrecognized  {get_unrecognized_folders.__doc__}\n'
+        )
+    )
+    parser.add_argument(
+        '--filter-type', '-t',
+        type=str,
+        choices=(*LINK_FILTERS.keys(), 'search'),
+        default='exact',
+        help='Type of pattern matching to use when filtering URLs',
+    )
+    parser.add_argument(
+        'filter_patterns',
+        nargs='*',
+        type=str,
+        default=None,
+        help='Update only URLs matching these filter patterns.'
+    )
+    parser.add_argument(
+        "--extract",
+        type=str,
+        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
+              This does not take precedence over the configuration",
+        default=""
+    )
+    command = parser.parse_args(args or ())

+    filter_patterns_str = None
+    if not command.filter_patterns:
+        filter_patterns_str = accept_stdin(stdin)

-if __name__ == "__main__":
-    main()
+    update(
+        resume=command.resume,
+        only_new=command.only_new,
+        index_only=command.index_only,
+        overwrite=command.overwrite,
+        filter_patterns_str=filter_patterns_str,
+        filter_patterns=command.filter_patterns,
+        filter_type=command.filter_type,
+        status=command.status,
+        after=command.after,
+        before=command.before,
+        out_dir=pwd or OUTPUT_DIR,
+        extractors=command.extract,
+    )
+    
+
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,241 +1,40 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox version'

 import sys
-import os
-import platform
-from pathlib import Path
-from collections.abc import Iterable
+import argparse

-import rich_click as click
+from typing import Optional, List, IO

-from archivebox.misc.util import docstring, enforce_types
+from ..main import version
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin


-@enforce_types
-def version(
-    quiet: bool = False,
-    binaries: Iterable[str] = (),
-) -> list[str]:
-    """Print the ArchiveBox version, debug metadata, and installed dependency versions"""
-
-    # fast path for just getting the version and exiting, dont do any slower imports
-    from archivebox.config.version import VERSION
-
-    print(VERSION)
-    if quiet or "--version" in sys.argv:
-        return []
-
-    from rich.panel import Panel
-    from rich.console import Console
-
-    from archivebox.config import CONSTANTS, DATA_DIR
-    from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
-    from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID, IN_DOCKER
-    from archivebox.config.paths import get_data_locations, get_code_locations
-    from archivebox.config.common import SHELL_CONFIG, STORAGE_CONFIG, SEARCH_BACKEND_CONFIG
-    from archivebox.misc.logging_util import printable_folder_status
-    from archivebox.config.configset import get_config
-
-    console = Console()
-    prnt = console.print
-
-    # Check if LDAP is enabled (simple config lookup)
-    config = get_config()
-    LDAP_ENABLED = config.get("LDAP_ENABLED", False)
-
-    p = platform.uname()
-    COMMIT_HASH = get_COMMIT_HASH()
-    prnt(
-        f"[dark_green]ArchiveBox[/dark_green] [dark_goldenrod]v{CONSTANTS.VERSION}[/dark_goldenrod]",
-        f"COMMIT_HASH={COMMIT_HASH[:7] if COMMIT_HASH else 'unknown'}",
-        f"BUILD_TIME={get_BUILD_TIME()}",
-    )
-    prnt(
-        f"IN_DOCKER={IN_DOCKER}",
-        f"IN_QEMU={SHELL_CONFIG.IN_QEMU}",
-        f"ARCH={p.machine}",
-        f"OS={p.system}",
-        f"PLATFORM={platform.platform()}",
-        f"PYTHON={sys.implementation.name.title()}" + (" (venv)" if CONSTANTS.IS_INSIDE_VENV else ""),
-    )
-
-    try:
-        OUTPUT_IS_REMOTE_FS = get_data_locations().DATA_DIR.is_mount or get_data_locations().ARCHIVE_DIR.is_mount
-    except Exception:
-        OUTPUT_IS_REMOTE_FS = False
-
-    try:
-        DATA_DIR_STAT = CONSTANTS.DATA_DIR.stat()
-        prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
-            f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}",
-            f"FS_PERMS={STORAGE_CONFIG.OUTPUT_PERMISSIONS}",
-            f"FS_ATOMIC={STORAGE_CONFIG.ENFORCE_ATOMIC_WRITES}",
-            f"FS_REMOTE={OUTPUT_IS_REMOTE_FS}",
-        )
-    except Exception:
-        prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
-        )
-
-    prnt(
-        f"DEBUG={SHELL_CONFIG.DEBUG}",
-        f"IS_TTY={SHELL_CONFIG.IS_TTY}",
-        f"SUDO={CONSTANTS.IS_ROOT}",
-        f"ID={CONSTANTS.MACHINE_ID}:{CONSTANTS.COLLECTION_ID}",
-        f"SEARCH_BACKEND={SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}",
-        f"LDAP={LDAP_ENABLED}",
-    )
-    prnt()
-
-    if not (os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
-        PANEL_TEXT = "\n".join(
-            (
-                "",
-                "[violet]Hint:[/violet] [green]cd[/green] into a collection [blue]DATA_DIR[/blue] and run [green]archivebox version[/green] again...",
-                "      [grey53]OR[/grey53] run [green]archivebox init[/green] to create a new collection in the current dir.",
-                "",
-                "      [i][grey53](this is [red]REQUIRED[/red] if you are opening a Github Issue to get help)[/grey53][/i]",
-                "",
-            ),
-        )
-        prnt(
-            Panel(
-                PANEL_TEXT,
-                expand=False,
-                border_style="grey53",
-                title="[red]:exclamation: No collection [blue]DATA_DIR[/blue] is currently active[/red]",
-                subtitle="Full version info is only available when inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
-            ),
-        )
-        prnt()
-        return []
-
-    prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
-    failures = []
-
-    # Setup Django before importing models
-    try:
-        from archivebox.config.django import setup_django
-
-        setup_django()
-
-        from archivebox.machine.models import Machine, Binary
-
-        machine = Machine.current()
-
-        # Get all binaries from the database with timeout protection
-        all_installed = (
-            Binary.objects.filter(
-                machine=machine,
-            )
-            .exclude(abspath="")
-            .exclude(abspath__isnull=True)
-            .order_by("name")
-        )
-
-        if not all_installed.exists():
-            prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
-        else:
-            for installed in all_installed:
-                # Skip if user specified specific binaries and this isn't one
-                if binaries and installed.name not in binaries:
-                    continue
-
-                if installed.is_valid:
-                    display_path = installed.abspath.replace(str(DATA_DIR), ".").replace(str(Path("~").expanduser()), "~")
-                    version_str = (installed.version or "unknown")[:15]
-                    provider = (installed.binprovider or "env")[:8]
-                    prnt(
-                        "",
-                        "[green]√[/green]",
-                        "",
-                        installed.name.ljust(18),
-                        version_str.ljust(16),
-                        provider.ljust(8),
-                        display_path,
-                        overflow="ignore",
-                        crop=False,
-                    )
-                else:
-                    prnt("", "[red]X[/red]", "", installed.name.ljust(18), "[grey53]not installed[/grey53]", overflow="ignore", crop=False)
-                    failures.append(installed.name)
-
-        # Show hint if no binaries are installed yet
-        has_any_installed = Binary.objects.filter(machine=machine).exclude(abspath="").exists()
-        if not has_any_installed:
-            prnt()
-            prnt("", "[grey53]Run [green]archivebox install[/green] to detect and install dependencies.[/grey53]")
-
-    except Exception as e:
-        # Handle database errors gracefully (locked, missing, etc.)
-        prnt()
-        prnt("", f"[yellow]Warning: Could not query binaries from database: {e}[/yellow]")
-        prnt("", "[grey53]Run [green]archivebox init[/green] and [green]archivebox install[/green] to set up dependencies.[/grey53]")
-
-    if not binaries:
-        # Show code and data locations
-        prnt()
-        prnt("[deep_sky_blue3][i] Code locations:[/deep_sky_blue3]")
-        try:
-            for name, path in get_code_locations().items():
-                if isinstance(name, str) and isinstance(path, dict):
-                    prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
-        except Exception as e:
-            prnt(f"  [red]Error getting code locations: {e}[/red]")
-
-        prnt()
-        if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
-            prnt("[bright_yellow][i] Data locations:[/bright_yellow]")
-            try:
-                for name, path in get_data_locations().items():
-                    if isinstance(name, str) and isinstance(path, dict):
-                        prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
-            except Exception as e:
-                prnt(f"  [red]Error getting data locations: {e}[/red]")
-
-            try:
-                from archivebox.misc.checks import check_data_dir_permissions
-
-                check_data_dir_permissions()
-            except Exception:
-                pass
-        else:
-            prnt()
-            prnt("[red][i] Data locations:[/red] (not in a data directory)")
-
-    prnt()
-
-    if failures:
-        prnt("[red]Error:[/red] [yellow]Failed to detect the following binaries:[/yellow]")
-        prnt(f"      [red]{', '.join(failures)}[/red]")
-        prnt()
-        prnt("[violet]Hint:[/violet] To install missing binaries automatically, run:")
-        prnt("      [green]archivebox install[/green]")
-        prnt()
-    return failures
-
-
-@click.command()
-@click.option(
-    "--quiet",
-    "-q",
-    is_flag=True,
-    help="Only print ArchiveBox version number and nothing else. (equivalent to archivebox --version)",
-)
-@click.option(
-    "--binaries",
-    "-b",
-    help="Select binaries to detect DEFAULT=curl,wget,git,yt-dlp,chrome,single-file,readability-extractor,postlight-parser,... (all)",
-)
@docstring(version.__doc__)
-def main(**kwargs):
-    failures = version(**kwargs)
-    if failures:
-        raise SystemExit(1)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=version.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--quiet', '-q',
+        action='store_true',
+        help='Only print ArchiveBox version number and nothing else.',
+    )
+    command = parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)
+    
+    version(
+        quiet=command.quiet,
+        out_dir=pwd or OUTPUT_DIR,
+    )


-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,44 +0,0 @@
-"""
-Shared CLI utilities for ArchiveBox commands.
-
-This module contains common utilities used across multiple CLI commands,
-extracted to avoid code duplication.
-"""
-
-__package__ = "archivebox.cli"
-
-
-def apply_filters(queryset, filter_kwargs: dict, limit: int | None = None):
-    """
-    Apply Django-style filters from CLI kwargs to a QuerySet.
-
-    Supports: --status=queued, --url__icontains=example, --id__in=uuid1,uuid2
-
-    Args:
-        queryset: Django QuerySet to filter
-        filter_kwargs: Dict of filter key-value pairs from CLI
-        limit: Optional limit on results
-
-    Returns:
-        Filtered QuerySet
-
-    Example:
-        queryset = Snapshot.objects.all()
-        filter_kwargs = {'status': 'queued', 'url__icontains': 'example.com'}
-        filtered = apply_filters(queryset, filter_kwargs, limit=10)
-    """
-    filters = {}
-    for key, value in filter_kwargs.items():
-        if value is None or key in ("limit", "offset"):
-            continue
-        # Handle CSV lists for __in filters
-        if key.endswith("__in") and isinstance(value, str):
-            value = [v.strip() for v in value.split(",")]
-        filters[key] = value
-
-    if filters:
-        queryset = queryset.filter(**filters)
-    if limit:
-        queryset = queryset[:limit]
-
-    return queryset
@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+
+__package__ = 'archivebox.cli'
+
+
+import os
+import sys
+import shutil
+import unittest
+from pathlib import Path
+
+from contextlib import contextmanager
+
+TEST_CONFIG = {
+    'USE_COLOR': 'False',
+    'SHOW_PROGRESS': 'False',
+
+    'OUTPUT_DIR': 'data.tests',
+    
+    'SAVE_ARCHIVE_DOT_ORG': 'False',
+    'SAVE_TITLE': 'False',
+    
+    'USE_CURL': 'False',
+    'USE_WGET': 'False',
+    'USE_GIT': 'False',
+    'USE_CHROME': 'False',
+    'USE_YOUTUBEDL': 'False',
+}
+
+OUTPUT_DIR = 'data.tests'
+os.environ.update(TEST_CONFIG)
+
+from ..main import init
+from ..index import load_main_index
+from ..config import (
+    SQL_INDEX_FILENAME,
+    JSON_INDEX_FILENAME,
+    HTML_INDEX_FILENAME,
+)
+
+from . import (
+    archivebox_init,
+    archivebox_add,
+    archivebox_remove,
+)
+
+HIDE_CLI_OUTPUT = True
+
+test_urls = '''
+https://example1.com/what/is/happening.html?what=1#how-about-this=1
+https://example2.com/what/is/happening/?what=1#how-about-this=1
+HTtpS://example3.com/what/is/happening/?what=1#how-about-this=1f
+https://example4.com/what/is/happening.html
+https://example5.com/
+https://example6.com
+
+<test>http://example7.com</test>
+[https://example8.com/what/is/this.php?what=1]
+[and http://example9.com?what=1&other=3#and-thing=2]
+<what>https://example10.com#and-thing=2 "</about>
+abc<this["https://subb.example11.com/what/is#and-thing=2?whoami=23&where=1"]that>def
+sdflkf[what](https://subb.example12.com/who/what.php?whoami=1#whatami=2)?am=hi
+example13.bada
+and example14.badb
+<or>htt://example15.badc</that>
+'''
+
+stdout = sys.stdout
+stderr = sys.stderr
+
+
+@contextmanager
+def output_hidden(show_failing=True):
+    if not HIDE_CLI_OUTPUT:
+        yield
+        return
+
+    sys.stdout = open('stdout.txt', 'w+', encoding='utf-8')
+    sys.stderr = open('stderr.txt', 'w+', encoding='utf-8')
+    try:
+        yield
+        sys.stdout.close()
+        sys.stderr.close()
+        sys.stdout = stdout
+        sys.stderr = stderr
+    except Exception:
+        sys.stdout.close()
+        sys.stderr.close()
+        sys.stdout = stdout
+        sys.stderr = stderr
+        if show_failing:
+            with open('stdout.txt', 'r', encoding='utf-8') as f:
+                print(f.read())
+            with open('stderr.txt', 'r', encoding='utf-8') as f:
+                print(f.read())
+        raise
+    finally:
+        os.remove('stdout.txt')
+        os.remove('stderr.txt')
+
+
+class TestInit(unittest.TestCase):
+    def setUp(self):
+        os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+    def tearDown(self):
+        shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
+
+    def test_basic_init(self):
+        with output_hidden():
+            archivebox_init.main([])
+
+        assert (Path(OUTPUT_DIR) / SQL_INDEX_FILENAME).exists()
+        assert (Path(OUTPUT_DIR) / JSON_INDEX_FILENAME).exists()
+        assert (Path(OUTPUT_DIR) / HTML_INDEX_FILENAME).exists()
+        assert len(load_main_index(out_dir=OUTPUT_DIR)) == 0
+
+    def test_conflicting_init(self):
+        with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
+            f.write('test')
+
+        try:
+            with output_hidden(show_failing=False):
+                archivebox_init.main([])
+            assert False, 'Init should have exited with an exception'
+        except SystemExit:
+            pass
+
+        assert not (Path(OUTPUT_DIR) / SQL_INDEX_FILENAME).exists()
+        assert not (Path(OUTPUT_DIR) / JSON_INDEX_FILENAME).exists()
+        assert not (Path(OUTPUT_DIR) / HTML_INDEX_FILENAME).exists()
+        try:
+            load_main_index(out_dir=OUTPUT_DIR)
+            assert False, 'load_main_index should raise an exception when no index is present'
+        except Exception:
+            pass
+
+    def test_no_dirty_state(self):
+        with output_hidden():
+            init()
+        shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
+        with output_hidden():
+            init()
+
+
+class TestAdd(unittest.TestCase):
+    def setUp(self):
+        os.makedirs(OUTPUT_DIR, exist_ok=True)
+        with output_hidden():
+            init()
+
+    def tearDown(self):
+        shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
+
+    def test_add_arg_url(self):
+        with output_hidden():
+            archivebox_add.main(['https://getpocket.com/users/nikisweeting/feed/all'])
+
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
+        assert len(all_links) == 30
+
+    def test_add_arg_file(self):
+        test_file = Path(OUTPUT_DIR) / 'test.txt'
+        with open(test_file, 'w+', encoding='utf') as f:
+            f.write(test_urls)
+
+        with output_hidden():
+            archivebox_add.main([test_file])
+
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
+        assert len(all_links) == 12
+        os.remove(test_file)
+
+    def test_add_stdin_url(self):
+        with output_hidden():
+            archivebox_add.main([], stdin=test_urls)
+
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
+        assert len(all_links) == 12
+
+
+class TestRemove(unittest.TestCase):
+    def setUp(self):
+        os.makedirs(OUTPUT_DIR, exist_ok=True)
+        with output_hidden():
+            init()
+            archivebox_add.main([], stdin=test_urls)
+
+    # def tearDown(self):
+        # shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
+
+
+    def test_remove_exact(self):
+        with output_hidden():
+            archivebox_remove.main(['--yes', '--delete', 'https://example5.com/'])
+
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
+        assert len(all_links) == 11
+
+    def test_remove_regex(self):
+        with output_hidden():
+            archivebox_remove.main(['--yes', '--delete', '--filter-type=regex', r'http(s)?:\/\/(.+\.)?(example\d\.com)'])
+
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
+        assert len(all_links) == 4
+
+    def test_remove_domain(self):
+        with output_hidden():
+            archivebox_remove.main(['--yes', '--delete', '--filter-type=domain', 'example5.com', 'example6.com'])
+
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
+        assert len(all_links) == 10
+
+    def test_remove_none(self):
+        try:
+            with output_hidden(show_failing=False):
+                archivebox_remove.main(['--yes', '--delete', 'https://doesntexist.com'])
+            assert False, 'Should raise if no URLs match'
+        except Exception:
+            pass
+
+
+if __name__ == '__main__':
+    if '--verbose' in sys.argv or '-v' in sys.argv:
+        HIDE_CLI_OUTPUT = False
+    
+    unittest.main()
@@ -1,105 +0,0 @@
-"""
-ArchiveBox config exports.
-
-This module provides backwards-compatible config exports for extractors
-and other modules that expect to import config values directly.
-"""
-
-__package__ = "archivebox.config"
-__order__ = 200
-
-from .paths import (
-    PACKAGE_DIR,
-    DATA_DIR,
-    ARCHIVE_DIR,
-)
-from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR  # noqa
-from .version import VERSION  # noqa
-
-
-###############################################################################
-# Config value exports for extractors
-# These provide backwards compatibility with extractors that import from ..config
-###############################################################################
-
-
-def _get_config():
-    """Lazy import to avoid circular imports."""
-    from .common import ARCHIVING_CONFIG, STORAGE_CONFIG
-
-    return ARCHIVING_CONFIG, STORAGE_CONFIG
-
-
-# Direct exports (evaluated at import time for backwards compat)
-# These are recalculated each time the module attribute is accessed
-
-
-def __getattr__(name: str):
-    """
-    Module-level __getattr__ for lazy config loading.
-
-    Only provides backwards compatibility for GENERIC/SHARED config.
-    Plugin-specific config (binaries, args, toggles) should come from plugin config.json files.
-    """
-
-    # Generic timeout settings (used by multiple plugins)
-    if name == "TIMEOUT":
-        cfg, _ = _get_config()
-        return cfg.TIMEOUT
-
-    # Generic SSL/Security settings (used by multiple plugins)
-    if name == "CHECK_SSL_VALIDITY":
-        cfg, _ = _get_config()
-        return cfg.CHECK_SSL_VALIDITY
-
-    # Generic storage settings (used by multiple plugins)
-    if name == "RESTRICT_FILE_NAMES":
-        _, storage = _get_config()
-        return storage.RESTRICT_FILE_NAMES
-
-    # Generic user agent / cookies (used by multiple plugins)
-    if name == "COOKIES_FILE":
-        cfg, _ = _get_config()
-        return cfg.COOKIES_FILE
-    if name == "USER_AGENT":
-        cfg, _ = _get_config()
-        return cfg.USER_AGENT
-
-    # Generic resolution settings (used by multiple plugins)
-    if name == "RESOLUTION":
-        cfg, _ = _get_config()
-        return cfg.RESOLUTION
-
-    # Allowlist/Denylist patterns (compiled regexes)
-    if name == "SAVE_ALLOWLIST_PTN":
-        cfg, _ = _get_config()
-        return cfg.SAVE_ALLOWLIST_PTNS
-    if name == "SAVE_DENYLIST_PTN":
-        cfg, _ = _get_config()
-        return cfg.SAVE_DENYLIST_PTNS
-
-    raise AttributeError(f"module 'archivebox.config' has no attribute '{name}'")
-
-
-# Re-export common config classes for direct imports
-def get_CONFIG():
-    """Get all config sections as a dict."""
-    from .common import (
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-    from .ldap import LDAP_CONFIG
-
-    return {
-        "SHELL_CONFIG": SHELL_CONFIG,
-        "STORAGE_CONFIG": STORAGE_CONFIG,
-        "GENERAL_CONFIG": GENERAL_CONFIG,
-        "SERVER_CONFIG": SERVER_CONFIG,
-        "ARCHIVING_CONFIG": ARCHIVING_CONFIG,
-        "SEARCHBACKEND_CONFIG": SEARCH_BACKEND_CONFIG,
-        "LDAP_CONFIG": LDAP_CONFIG,
-    }
@@ -1,305 +0,0 @@
-__package__ = "archivebox.config"
-
-import os
-import json
-from typing import Any
-
-from pathlib import Path
-from configparser import ConfigParser
-
-from benedict import benedict
-
-
-from archivebox.config.constants import CONSTANTS
-
-from archivebox.misc.logging import stderr
-
-
-class CaseConfigParser(ConfigParser):
-    def optionxform(self, optionstr: str) -> str:
-        return optionstr
-
-
-def get_real_name(key: str) -> str:
-    """get the up-to-date canonical name for a given old alias or current key"""
-    # Config aliases are no longer used with the simplified config system
-    # Just return the key as-is since we no longer have a complex alias mapping
-    return key
-
-
-def load_config_val(
-    key: str,
-    default: Any = None,
-    type: type | None = None,
-    aliases: tuple[str, ...] | None = None,
-    config: benedict | None = None,
-    env_vars: os._Environ | None = None,
-    config_file_vars: dict[str, str] | None = None,
-) -> Any:
-    """parse bool, int, and str key=value pairs from env"""
-
-    assert isinstance(config, dict)
-
-    is_read_only = type is None
-    if is_read_only:
-        if callable(default):
-            return default(config)
-        return default
-
-    # get value from environment variables or config files
-    config_keys_to_check = (key, *(aliases or ()))
-    val = None
-    for key in config_keys_to_check:
-        if env_vars:
-            val = env_vars.get(key)
-            if val:
-                break
-
-        if config_file_vars:
-            val = config_file_vars.get(key)
-            if val:
-                break
-
-    is_unset = val is None
-    if is_unset:
-        if callable(default):
-            return default(config)
-        return default
-
-    assert isinstance(val, str)
-
-    # calculate value based on expected type
-    BOOL_TRUEIES = ("true", "yes", "1")
-    BOOL_FALSEIES = ("false", "no", "0")
-
-    if type is bool:
-        if val.lower() in BOOL_TRUEIES:
-            return True
-        elif val.lower() in BOOL_FALSEIES:
-            return False
-        else:
-            raise ValueError(f"Invalid configuration option {key}={val} (expected a boolean: True/False)")
-
-    elif type is str:
-        if val.lower() in (*BOOL_TRUEIES, *BOOL_FALSEIES):
-            raise ValueError(f"Invalid configuration option {key}={val} (expected a string, but value looks like a boolean)")
-        return val.strip()
-
-    elif type is int:
-        if not val.strip().isdigit():
-            raise ValueError(f"Invalid configuration option {key}={val} (expected an integer)")
-        return int(val.strip())
-
-    elif type is list or type is dict:
-        return json.loads(val)
-
-    elif type is Path:
-        return Path(val)
-
-    raise Exception("Config values can only be str, bool, int, or json")
-
-
-def load_config_file() -> benedict | None:
-    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
-
-    config_path = CONSTANTS.CONFIG_FILE
-    if os.access(config_path, os.R_OK):
-        config_file = CaseConfigParser()
-        config_file.read(config_path)
-        # flatten into one namespace
-        config_file_vars = benedict({key.upper(): val for section, options in config_file.items() for key, val in options.items()})
-        # print('[i] Loaded config file', os.path.abspath(config_path))
-        # print(config_file_vars)
-        return config_file_vars
-    return None
-
-
-class PluginConfigSection:
-    """Pseudo-section for all plugin config keys written to [PLUGINS] section in ArchiveBox.conf"""
-
-    toml_section_header = "PLUGINS"
-
-    def __init__(self, key: str):
-        self._key = key
-
-    def __getattr__(self, name: str) -> Any:
-        # Allow hasattr checks to pass for the key
-        if name == self._key:
-            return None
-        raise AttributeError(f"PluginConfigSection has no attribute '{name}'")
-
-    def update_in_place(self, warn: bool = True, persist: bool = False, **kwargs):
-        """No-op update since plugins read config dynamically via get_config()."""
-        pass
-
-
-def section_for_key(key: str) -> Any:
-    """Find the config section containing a given key."""
-    from archivebox.config.common import (
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-
-    # First check core config sections
-    for section in [
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    ]:
-        if hasattr(section, key):
-            return section
-
-    # Check if this is a plugin config key
-    from archivebox.hooks import discover_plugin_configs
-
-    plugin_configs = discover_plugin_configs()
-    for plugin_name, schema in plugin_configs.items():
-        if "properties" in schema and key in schema["properties"]:
-            # All plugin config goes to [PLUGINS] section
-            return PluginConfigSection(key)
-
-    raise ValueError(f"No config section found for key: {key}")
-
-
-def write_config_file(config: dict[str, str]) -> benedict:
-    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
-
-    from archivebox.misc.system import atomic_write
-
-    CONFIG_HEADER = """# This is the config file for your ArchiveBox collection.
-    #
-    # You can add options here manually in INI format, or automatically by running:
-    #    archivebox config --set KEY=VALUE
-    #
-    # If you modify this file manually, make sure to update your archive after by running:
-    #    archivebox init
-    #
-    # A list of all possible config with documentation and examples can be found here:
-    #    https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration
-
-    """
-
-    config_path = CONSTANTS.CONFIG_FILE
-
-    if not os.access(config_path, os.F_OK):
-        atomic_write(config_path, CONFIG_HEADER)
-
-    config_file = CaseConfigParser()
-    config_file.read(config_path)
-
-    with open(config_path, encoding="utf-8") as old:
-        atomic_write(f"{config_path}.bak", old.read())
-
-    # Set up sections in empty config file
-    for key, val in config.items():
-        section = section_for_key(key)
-        assert section is not None
-
-        if not hasattr(section, "toml_section_header"):
-            raise ValueError(f"{key} is read-only (defined in {type(section).__module__}.{type(section).__name__}). Refusing to set.")
-
-        section_name = section.toml_section_header
-
-        if section_name in config_file:
-            existing_config = dict(config_file[section_name])
-        else:
-            existing_config = {}
-
-        config_file[section_name] = benedict({**existing_config, key: val})
-        section.update_in_place(warn=False, persist=False, **{key: val})
-
-    with open(config_path, "w+", encoding="utf-8") as new:
-        config_file.write(new)
-
-    updated_config = {}
-    try:
-        # validate the updated_config by attempting to re-parse it
-        from archivebox.config.configset import get_flat_config
-
-        updated_config = {**load_all_config(), **get_flat_config()}
-    except BaseException:  # lgtm [py/catch-base-exception]
-        # something went horribly wrong, revert to the previous version
-        with open(f"{config_path}.bak", encoding="utf-8") as old:
-            atomic_write(config_path, old.read())
-
-        raise
-
-    if os.access(f"{config_path}.bak", os.F_OK):
-        os.remove(f"{config_path}.bak")
-
-    return benedict({key.upper(): updated_config.get(key.upper()) for key in config.keys()})
-
-
-def load_config(
-    defaults: dict[str, Any],
-    config: benedict | None = None,
-    out_dir: str | None = None,
-    env_vars: os._Environ | None = None,
-    config_file_vars: dict[str, str] | None = None,
-) -> benedict:
-
-    env_vars = env_vars or os.environ
-    config_file_vars = config_file_vars or load_config_file()
-
-    extended_config = benedict(config.copy() if config else {})
-    for key, default in defaults.items():
-        try:
-            # print('LOADING CONFIG KEY:', key, 'DEFAULT=', default)
-            extended_config[key] = load_config_val(
-                key,
-                default=default["default"],
-                type=default.get("type"),
-                aliases=default.get("aliases"),
-                config=extended_config,
-                env_vars=env_vars,
-                config_file_vars=config_file_vars,
-            )
-        except KeyboardInterrupt:
-            raise SystemExit(0)
-        except Exception as e:
-            stderr()
-            stderr(f"[X] Error while loading configuration value: {key}", color="red", config=extended_config)
-            stderr(f"    {e.__class__.__name__}: {e}")
-            stderr()
-            stderr("    Check your config for mistakes and try again (your archive data is unaffected).")
-            stderr()
-            stderr("    For config documentation and examples see:")
-            stderr("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration")
-            stderr()
-            # raise
-            # raise SystemExit(2)
-
-    return benedict(extended_config)
-
-
-def load_all_config():
-    """Load all config sections and return as a flat dict."""
-    from archivebox.config.common import (
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-
-    flat_config = benedict()
-
-    for config_section in [
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    ]:
-        flat_config.update(dict(config_section))
-
-    return flat_config
@@ -1,315 +0,0 @@
-__package__ = "archivebox.config"
-
-import re
-import secrets
-import sys
-import shutil
-from typing import ClassVar
-from pathlib import Path
-
-from rich.console import Console
-from pydantic import Field, field_validator
-
-from archivebox.config.configset import BaseConfigSet
-
-from .constants import CONSTANTS
-from .version import get_COMMIT_HASH, get_BUILD_TIME, VERSION
-from .permissions import IN_DOCKER
-
-###################### Config ##########################
-
-_STDOUT_CONSOLE = Console()
-_STDERR_CONSOLE = Console(stderr=True)
-
-
-def rprint(*args, file=None, **kwargs):
-    console = _STDERR_CONSOLE if file is sys.stderr else _STDOUT_CONSOLE
-    console.print(*args, **kwargs)
-
-
-class ShellConfig(BaseConfigSet):
-    toml_section_header: str = "SHELL_CONFIG"
-
-    DEBUG: bool = Field(default="--debug" in sys.argv)
-
-    IS_TTY: bool = Field(default=sys.stdout.isatty())
-    USE_COLOR: bool = Field(default=sys.stdout.isatty())
-    SHOW_PROGRESS: bool = Field(default=sys.stdout.isatty())
-
-    IN_DOCKER: bool = Field(default=IN_DOCKER)
-    IN_QEMU: bool = Field(default=False)
-
-    ANSI: dict[str, str] = Field(
-        default_factory=lambda: CONSTANTS.DEFAULT_CLI_COLORS if sys.stdout.isatty() else CONSTANTS.DISABLED_CLI_COLORS,
-    )
-
-    @property
-    def TERM_WIDTH(self) -> int:
-        if not self.IS_TTY:
-            return 200
-        return shutil.get_terminal_size((140, 10)).columns
-
-    @property
-    def COMMIT_HASH(self) -> str | None:
-        return get_COMMIT_HASH()
-
-    @property
-    def BUILD_TIME(self) -> str:
-        return get_BUILD_TIME()
-
-
-SHELL_CONFIG = ShellConfig()
-
-
-class StorageConfig(BaseConfigSet):
-    toml_section_header: str = "STORAGE_CONFIG"
-
-    # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
-    # must be a short path due to unix path length restrictions for socket files (<100 chars)
-    # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
-    TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR)
-
-    # LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
-    # must be able to contain executable binaries (up to 5GB size)
-    # should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
-    LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR)
-
-    # LIB_BIN_DIR is where all installed binaries are symlinked for easy PATH management
-    # Derived from LIB_DIR / 'bin', should be prepended to PATH for all hook executions
-    LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR)
-
-    # CUSTOM_TEMPLATES_DIR allows users to override default templates
-    # defaults to DATA_DIR / 'user_templates' but can be configured
-    CUSTOM_TEMPLATES_DIR: Path = Field(default=CONSTANTS.CUSTOM_TEMPLATES_DIR)
-
-    OUTPUT_PERMISSIONS: str = Field(default="644")
-    RESTRICT_FILE_NAMES: str = Field(default="windows")
-    ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
-
-    # not supposed to be user settable:
-    DIR_OUTPUT_PERMISSIONS: str = Field(default="755")  # computed from OUTPUT_PERMISSIONS
-
-
-STORAGE_CONFIG = StorageConfig()
-
-
-class GeneralConfig(BaseConfigSet):
-    toml_section_header: str = "GENERAL_CONFIG"
-
-    TAG_SEPARATOR_PATTERN: str = Field(default=r"[,]")
-
-
-GENERAL_CONFIG = GeneralConfig()
-
-
-class ServerConfig(BaseConfigSet):
-    toml_section_header: str = "SERVER_CONFIG"
-
-    SERVER_SECURITY_MODES: ClassVar[tuple[str, ...]] = (
-        "safe-subdomains-fullreplay",
-        "safe-onedomain-nojsreplay",
-        "unsafe-onedomain-noadmin",
-        "danger-onedomain-fullreplay",
-    )
-
-    SECRET_KEY: str = Field(default_factory=lambda: "".join(secrets.choice("abcdefghijklmnopqrstuvwxyz0123456789_") for _ in range(50)))
-    BIND_ADDR: str = Field(default="127.0.0.1:8000")
-    LISTEN_HOST: str = Field(default="archivebox.localhost:8000")
-    ADMIN_BASE_URL: str = Field(default="")
-    ARCHIVE_BASE_URL: str = Field(default="")
-    ALLOWED_HOSTS: str = Field(default="*")
-    CSRF_TRUSTED_ORIGINS: str = Field(default="http://admin.archivebox.localhost:8000")
-    SERVER_SECURITY_MODE: str = Field(default="safe-subdomains-fullreplay")
-
-    SNAPSHOTS_PER_PAGE: int = Field(default=40)
-    PREVIEW_ORIGINALS: bool = Field(default=True)
-    FOOTER_INFO: str = Field(
-        default="Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.",
-    )
-    # CUSTOM_TEMPLATES_DIR: Path          = Field(default=None)  # this is now a constant
-
-    PUBLIC_INDEX: bool = Field(default=True)
-    PUBLIC_SNAPSHOTS: bool = Field(default=True)
-    PUBLIC_ADD_VIEW: bool = Field(default=False)
-
-    ADMIN_USERNAME: str | None = Field(default=None)
-    ADMIN_PASSWORD: str | None = Field(default=None)
-
-    REVERSE_PROXY_USER_HEADER: str = Field(default="Remote-User")
-    REVERSE_PROXY_WHITELIST: str = Field(default="")
-    LOGOUT_REDIRECT_URL: str = Field(default="/")
-
-    @field_validator("SERVER_SECURITY_MODE", mode="after")
-    def validate_server_security_mode(cls, v: str) -> str:
-        mode = (v or "").strip().lower()
-        if mode not in cls.SERVER_SECURITY_MODES:
-            raise ValueError(f"SERVER_SECURITY_MODE must be one of: {', '.join(cls.SERVER_SECURITY_MODES)}")
-        return mode
-
-    @property
-    def USES_SUBDOMAIN_ROUTING(self) -> bool:
-        return self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay"
-
-    @property
-    def ENABLES_FULL_JS_REPLAY(self) -> bool:
-        return self.SERVER_SECURITY_MODE in (
-            "safe-subdomains-fullreplay",
-            "unsafe-onedomain-noadmin",
-            "danger-onedomain-fullreplay",
-        )
-
-    @property
-    def CONTROL_PLANE_ENABLED(self) -> bool:
-        return self.SERVER_SECURITY_MODE != "unsafe-onedomain-noadmin"
-
-    @property
-    def BLOCK_UNSAFE_METHODS(self) -> bool:
-        return self.SERVER_SECURITY_MODE == "unsafe-onedomain-noadmin"
-
-    @property
-    def SHOULD_NEUTER_RISKY_REPLAY(self) -> bool:
-        return self.SERVER_SECURITY_MODE == "safe-onedomain-nojsreplay"
-
-    @property
-    def IS_UNSAFE_MODE(self) -> bool:
-        return self.SERVER_SECURITY_MODE == "unsafe-onedomain-noadmin"
-
-    @property
-    def IS_DANGEROUS_MODE(self) -> bool:
-        return self.SERVER_SECURITY_MODE == "danger-onedomain-fullreplay"
-
-    @property
-    def IS_LOWER_SECURITY_MODE(self) -> bool:
-        return self.SERVER_SECURITY_MODE in (
-            "unsafe-onedomain-noadmin",
-            "danger-onedomain-fullreplay",
-        )
-
-
-SERVER_CONFIG = ServerConfig()
-
-
-def _print_server_security_mode_warning() -> None:
-    if not SERVER_CONFIG.IS_LOWER_SECURITY_MODE:
-        return
-
-    rprint(
-        f"[yellow][!] WARNING: ArchiveBox is running with SERVER_SECURITY_MODE={SERVER_CONFIG.SERVER_SECURITY_MODE}[/yellow]",
-        file=sys.stderr,
-    )
-    rprint(
-        "[yellow]    Archived pages may share an origin with privileged app routes in this mode.[/yellow]",
-        file=sys.stderr,
-    )
-    rprint(
-        "[yellow]    To switch to the safer isolated setup:[/yellow]",
-        file=sys.stderr,
-    )
-    rprint(
-        "[yellow]    1. Set SERVER_SECURITY_MODE=safe-subdomains-fullreplay[/yellow]",
-        file=sys.stderr,
-    )
-    rprint(
-        "[yellow]    2. Point *.archivebox.localhost (or your chosen base domain) at this server[/yellow]",
-        file=sys.stderr,
-    )
-    rprint(
-        "[yellow]    3. Configure wildcard DNS/TLS or your reverse proxy so admin., web., api., and snapshot subdomains resolve[/yellow]",
-        file=sys.stderr,
-    )
-
-
-_print_server_security_mode_warning()
-
-
-class ArchivingConfig(BaseConfigSet):
-    toml_section_header: str = "ARCHIVING_CONFIG"
-
-    ONLY_NEW: bool = Field(default=True)
-    OVERWRITE: bool = Field(default=False)
-
-    TIMEOUT: int = Field(default=60)
-    MAX_URL_ATTEMPTS: int = Field(default=50)
-
-    RESOLUTION: str = Field(default="1440,2000")
-    CHECK_SSL_VALIDITY: bool = Field(default=True)
-    USER_AGENT: str = Field(
-        default=f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)",
-    )
-    COOKIES_FILE: Path | None = Field(default=None)
-
-    URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
-    URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")
-
-    SAVE_ALLOWLIST: dict[str, list[str]] = Field(default={})  # mapping of regex patterns to list of archive methods
-    SAVE_DENYLIST: dict[str, list[str]] = Field(default={})
-
-    DEFAULT_PERSONA: str = Field(default="Default")
-
-    def warn_if_invalid(self) -> None:
-        if int(self.TIMEOUT) < 5:
-            rprint(f"[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={self.TIMEOUT} seconds)[/red]", file=sys.stderr)
-            rprint("    You must allow *at least* 5 seconds for indexing and archive methods to run successfully.", file=sys.stderr)
-            rprint("    (Setting it to somewhere between 30 and 3000 seconds is recommended)", file=sys.stderr)
-            rprint(file=sys.stderr)
-            rprint("    If you want to make ArchiveBox run faster, disable specific archive methods instead:", file=sys.stderr)
-            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles", file=sys.stderr)
-            rprint(file=sys.stderr)
-
-    @field_validator("CHECK_SSL_VALIDITY", mode="after")
-    def validate_check_ssl_validity(cls, v):
-        """SIDE EFFECT: disable "you really shouldnt disable ssl" warnings emitted by requests"""
-        if not v:
-            import urllib3
-
-            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-        return v
-
-    @property
-    def URL_ALLOWLIST_PTN(self) -> re.Pattern | None:
-        return re.compile(self.URL_ALLOWLIST, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS) if self.URL_ALLOWLIST else None
-
-    @property
-    def URL_DENYLIST_PTN(self) -> re.Pattern:
-        return re.compile(self.URL_DENYLIST, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS)
-
-    @property
-    def SAVE_ALLOWLIST_PTNS(self) -> dict[re.Pattern, list[str]]:
-        return (
-            {
-                # regexp: methods list
-                re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val
-                for key, val in self.SAVE_ALLOWLIST.items()
-            }
-            if self.SAVE_ALLOWLIST
-            else {}
-        )
-
-    @property
-    def SAVE_DENYLIST_PTNS(self) -> dict[re.Pattern, list[str]]:
-        return (
-            {
-                # regexp: methods list
-                re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val
-                for key, val in self.SAVE_DENYLIST.items()
-            }
-            if self.SAVE_DENYLIST
-            else {}
-        )
-
-
-ARCHIVING_CONFIG = ArchivingConfig()
-ARCHIVING_CONFIG.warn_if_invalid()
-
-
-class SearchBackendConfig(BaseConfigSet):
-    toml_section_header: str = "SEARCH_BACKEND_CONFIG"
-
-    USE_INDEXING_BACKEND: bool = Field(default=True)
-    USE_SEARCHING_BACKEND: bool = Field(default=True)
-
-    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep")
-    SEARCH_PROCESS_HTML: bool = Field(default=True)
-
-
-SEARCH_BACKEND_CONFIG = SearchBackendConfig()
@@ -1,428 +0,0 @@
-"""
-Simplified config system for ArchiveBox.
-
-This replaces the complex abx_spec_config/base_configset.py with a simpler
-approach that still supports environment variables, config files, and
-per-object overrides.
-"""
-
-__package__ = "archivebox.config"
-
-import os
-import json
-from pathlib import Path
-from typing import Any
-from configparser import ConfigParser
-
-from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict
-
-
-class CaseConfigParser(ConfigParser):
-    def optionxform(self, optionstr: str) -> str:
-        return optionstr
-
-
-class IniConfigSettingsSource(PydanticBaseSettingsSource):
-    """
-    Custom settings source that reads from ArchiveBox.conf (INI format).
-    Flattens all sections into a single namespace.
-    """
-
-    def get_field_value(self, field: Any, field_name: str) -> tuple[Any, str, bool]:
-        config_vals = self._load_config_file()
-        field_value = config_vals.get(field_name.upper())
-        return field_value, field_name, False
-
-    def __call__(self) -> dict[str, Any]:
-        return self._load_config_file()
-
-    def _load_config_file(self) -> dict[str, Any]:
-        try:
-            from archivebox.config.constants import CONSTANTS
-
-            config_path = CONSTANTS.CONFIG_FILE
-        except ImportError:
-            return {}
-
-        if not config_path.exists():
-            return {}
-
-        parser = CaseConfigParser()
-        parser.read(config_path)
-
-        # Flatten all sections into single namespace (ignore section headers)
-        return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
-
-
-class BaseConfigSet(BaseSettings):
-    """
-    Base class for config sections.
-
-    Automatically loads values from (highest to lowest priority):
-    1. Environment variables
-    2. ArchiveBox.conf file (INI format, flattened)
-    3. Default values
-
-    Subclasses define fields with defaults and types:
-
-        class ShellConfig(BaseConfigSet):
-            DEBUG: bool = Field(default=False)
-            USE_COLOR: bool = Field(default=True)
-    """
-
-    model_config = SettingsConfigDict(
-        env_prefix="",
-        extra="ignore",
-        validate_default=True,
-    )
-
-    @classmethod
-    def settings_customise_sources(
-        cls,
-        settings_cls: type[BaseSettings],
-        init_settings: PydanticBaseSettingsSource,
-        env_settings: PydanticBaseSettingsSource,
-        dotenv_settings: PydanticBaseSettingsSource,
-        file_secret_settings: PydanticBaseSettingsSource,
-    ) -> tuple[PydanticBaseSettingsSource, ...]:
-        """
-        Define the order of settings sources (first = highest priority).
-        """
-        return (
-            init_settings,  # 1. Passed to __init__
-            env_settings,  # 2. Environment variables
-            IniConfigSettingsSource(settings_cls),  # 3. ArchiveBox.conf file
-            # dotenv_settings,       # Skip .env files
-            # file_secret_settings,  # Skip secrets files
-        )
-
-    @classmethod
-    def load_from_file(cls, config_path: Path) -> dict[str, str]:
-        """Load config values from INI file."""
-        if not config_path.exists():
-            return {}
-
-        parser = CaseConfigParser()
-        parser.read(config_path)
-
-        # Flatten all sections into single namespace
-        return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
-
-    def update_in_place(self, warn: bool = True, persist: bool = False, **kwargs) -> None:
-        """
-        Update config values in place.
-
-        This allows runtime updates to config without reloading.
-        """
-        for key, value in kwargs.items():
-            if hasattr(self, key):
-                # Use object.__setattr__ to bypass pydantic's frozen model
-                object.__setattr__(self, key, value)
-
-
-def get_config(
-    defaults: dict | None = None,
-    persona: Any = None,
-    user: Any = None,
-    crawl: Any = None,
-    snapshot: Any = None,
-    archiveresult: Any = None,
-    machine: Any = None,
-) -> dict[str, Any]:
-    """
-    Get merged config from all sources.
-
-    Priority (highest to lowest):
-    1. Per-snapshot config (snapshot.config JSON field)
-    2. Per-crawl config (crawl.config JSON field)
-    3. Per-user config (user.config JSON field)
-    4. Per-persona config (persona.get_derived_config() - includes CHROME_USER_DATA_DIR etc.)
-    5. Environment variables
-    6. Per-machine config (machine.config JSON field - resolved binary paths)
-    7. Config file (ArchiveBox.conf)
-    8. Plugin schema defaults (config.json)
-    9. Core config defaults
-
-    Args:
-        defaults: Default values to start with
-        persona: Persona object (provides derived paths like CHROME_USER_DATA_DIR)
-        user: User object with config JSON field
-        crawl: Crawl object with config JSON field
-        snapshot: Snapshot object with config JSON field
-        archiveresult: ArchiveResult object (auto-fetches snapshot)
-        machine: Machine object with config JSON field (defaults to Machine.current())
-
-    Note: Objects are auto-fetched from relationships if not provided:
-        - snapshot auto-fetched from archiveresult.snapshot
-        - crawl auto-fetched from snapshot.crawl
-        - user auto-fetched from crawl.created_by
-
-    Returns:
-        Merged config dict
-    """
-    # Auto-fetch related objects from relationships
-    if snapshot is None and archiveresult and hasattr(archiveresult, "snapshot"):
-        snapshot = archiveresult.snapshot
-
-    if crawl is None and snapshot and hasattr(snapshot, "crawl"):
-        crawl = snapshot.crawl
-
-    if user is None and crawl and hasattr(crawl, "created_by"):
-        user = crawl.created_by
-
-    if persona is None and crawl is not None:
-        from archivebox.personas.models import Persona
-
-        persona_id = getattr(crawl, "persona_id", None)
-        if persona_id:
-            persona = Persona.objects.filter(id=persona_id).first()
-            if persona is None:
-                raise Persona.DoesNotExist(f"Crawl {getattr(crawl, 'id', None)} references missing Persona {persona_id}")
-
-        if persona is None:
-            crawl_config = getattr(crawl, "config", None) or {}
-            default_persona_name = str(crawl_config.get("DEFAULT_PERSONA") or "").strip()
-            if default_persona_name:
-                persona, _ = Persona.objects.get_or_create(name=default_persona_name or "Default")
-                persona.ensure_dirs()
-    from archivebox.config.constants import CONSTANTS
-    from archivebox.config.common import (
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-
-    # Start with defaults
-    config = dict(defaults or {})
-
-    # Add plugin config defaults from JSONSchema config.json files
-    try:
-        from archivebox.hooks import get_config_defaults_from_plugins
-
-        plugin_defaults = get_config_defaults_from_plugins()
-        config.update(plugin_defaults)
-    except ImportError:
-        pass  # hooks not available yet during early startup
-
-    # Add all core config sections
-    config.update(dict(SHELL_CONFIG))
-    config.update(dict(STORAGE_CONFIG))
-    config.update(dict(GENERAL_CONFIG))
-    config.update(dict(SERVER_CONFIG))
-    config.update(dict(ARCHIVING_CONFIG))
-    config.update(dict(SEARCH_BACKEND_CONFIG))
-
-    # Load from archivebox.config.file
-    config_file = CONSTANTS.CONFIG_FILE
-    if config_file.exists():
-        file_config = BaseConfigSet.load_from_file(config_file)
-        config.update(file_config)
-
-    # Apply machine config overrides (cached binary paths, etc.)
-    if machine is None:
-        # Default to current machine if not provided
-        try:
-            from archivebox.machine.models import Machine
-
-            machine = Machine.current()
-        except Exception:
-            pass  # Machine might not be available during early init
-
-    if machine and hasattr(machine, "config") and machine.config:
-        config.update(machine.config)
-
-    # Override with environment variables (for keys that exist in config)
-    for key in config:
-        env_val = os.environ.get(key)
-        if env_val is not None:
-            config[key] = _parse_env_value(env_val, config.get(key))
-
-    # Also add NEW environment variables (not yet in config)
-    # This is important for worker subprocesses that receive config via Process.env
-    for key, value in os.environ.items():
-        if key.isupper() and key not in config:  # Only uppercase keys (config convention)
-            config[key] = _parse_env_value(value, None)
-
-    # Also check plugin config aliases in environment
-    try:
-        from archivebox.hooks import discover_plugin_configs
-
-        plugin_configs = discover_plugin_configs()
-        for plugin_name, schema in plugin_configs.items():
-            for key, prop_schema in schema.get("properties", {}).items():
-                # Check x-aliases
-                for alias in prop_schema.get("x-aliases", []):
-                    if alias in os.environ and key not in os.environ:
-                        config[key] = _parse_env_value(os.environ[alias], config.get(key))
-                        break
-                # Check x-fallback
-                fallback = prop_schema.get("x-fallback")
-                if fallback and fallback in config and key not in config:
-                    config[key] = config[fallback]
-    except ImportError:
-        pass
-
-    # Apply persona config overrides (includes derived paths like CHROME_USER_DATA_DIR)
-    if persona and hasattr(persona, "get_derived_config"):
-        config.update(persona.get_derived_config())
-
-    # Apply user config overrides
-    if user and hasattr(user, "config") and user.config:
-        config.update(user.config)
-
-    # Apply crawl config overrides
-    if crawl and hasattr(crawl, "config") and crawl.config:
-        config.update(crawl.config)
-
-    # Add crawl path aliases for hooks that need shared crawl state.
-    if crawl and hasattr(crawl, "output_dir"):
-        config["CRAWL_OUTPUT_DIR"] = str(crawl.output_dir)
-        config["CRAWL_DIR"] = str(crawl.output_dir)
-        config["CRAWL_ID"] = str(getattr(crawl, "id", "")) if getattr(crawl, "id", None) else config.get("CRAWL_ID")
-
-    # Apply snapshot config overrides (highest priority)
-    if snapshot and hasattr(snapshot, "config") and snapshot.config:
-        config.update(snapshot.config)
-
-    if snapshot:
-        config["SNAPSHOT_ID"] = str(getattr(snapshot, "id", "")) if getattr(snapshot, "id", None) else config.get("SNAPSHOT_ID")
-        config["SNAPSHOT_DEPTH"] = int(getattr(snapshot, "depth", 0) or 0)
-        if hasattr(snapshot, "output_dir"):
-            config["SNAP_DIR"] = str(snapshot.output_dir)
-        if getattr(snapshot, "crawl_id", None):
-            config["CRAWL_ID"] = str(snapshot.crawl_id)
-
-    # Normalize all aliases to canonical names (after all sources merged)
-    # This handles aliases that came from user/crawl/snapshot configs, not just env
-    try:
-        from archivebox.hooks import discover_plugin_configs
-
-        plugin_configs = discover_plugin_configs()
-        aliases_to_normalize = {}  # {alias_key: canonical_key}
-
-        # Build alias mapping from all plugin schemas
-        for plugin_name, schema in plugin_configs.items():
-            for canonical_key, prop_schema in schema.get("properties", {}).items():
-                for alias in prop_schema.get("x-aliases", []):
-                    aliases_to_normalize[alias] = canonical_key
-
-        # Normalize: copy alias values to canonical keys (aliases take precedence)
-        for alias_key, canonical_key in aliases_to_normalize.items():
-            if alias_key in config:
-                # Alias exists - copy to canonical key (overwriting any default)
-                config[canonical_key] = config[alias_key]
-                # Remove alias from config to keep it clean
-                del config[alias_key]
-    except ImportError:
-        pass
-
-    if not config.get("DATA_DIR"):
-        config["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
-    config["ABX_RUNTIME"] = "archivebox"
-
-    return config
-
-
-def get_flat_config() -> dict[str, Any]:
-    """
-    Get a flat dictionary of all config values.
-
-    Replaces abx.pm.hook.get_FLAT_CONFIG()
-    """
-    return get_config()
-
-
-def get_all_configs() -> dict[str, BaseConfigSet]:
-    """
-    Get all config section objects as a dictionary.
-
-    Replaces abx.pm.hook.get_CONFIGS()
-    """
-    from archivebox.config.common import (
-        SHELL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-
-    return {
-        "SHELL_CONFIG": SHELL_CONFIG,
-        "SERVER_CONFIG": SERVER_CONFIG,
-        "ARCHIVING_CONFIG": ARCHIVING_CONFIG,
-        "SEARCH_BACKEND_CONFIG": SEARCH_BACKEND_CONFIG,
-    }
-
-
-def _parse_env_value(value: str, default: Any = None) -> Any:
-    """Parse an environment variable value based on expected type."""
-    if default is None:
-        # Try to guess the type
-        if value.lower() in ("true", "false", "yes", "no", "1", "0"):
-            return value.lower() in ("true", "yes", "1")
-        try:
-            return int(value)
-        except ValueError:
-            pass
-        try:
-            return json.loads(value)
-        except (json.JSONDecodeError, ValueError):
-            pass
-        return value
-
-    # Parse based on default's type
-    if isinstance(default, bool):
-        return value.lower() in ("true", "yes", "1")
-    elif isinstance(default, int):
-        return int(value)
-    elif isinstance(default, float):
-        return float(value)
-    elif isinstance(default, (list, dict)):
-        return json.loads(value)
-    elif isinstance(default, Path):
-        return Path(value)
-    else:
-        return value
-
-
-# Default worker concurrency settings
-DEFAULT_WORKER_CONCURRENCY = {
-    "crawl": 2,
-    "snapshot": 3,
-    "wget": 2,
-    "ytdlp": 2,
-    "screenshot": 3,
-    "singlefile": 2,
-    "title": 5,
-    "favicon": 5,
-    "headers": 5,
-    "archivedotorg": 2,
-    "readability": 3,
-    "mercury": 3,
-    "git": 2,
-    "pdf": 2,
-    "dom": 3,
-}
-
-
-def get_worker_concurrency() -> dict[str, int]:
-    """
-    Get worker concurrency settings.
-
-    Can be configured via WORKER_CONCURRENCY env var as JSON dict.
-    """
-    config = get_config()
-
-    # Start with defaults
-    concurrency = DEFAULT_WORKER_CONCURRENCY.copy()
-
-    # Override with config
-    if "WORKER_CONCURRENCY" in config:
-        custom = config["WORKER_CONCURRENCY"]
-        if isinstance(custom, str):
-            custom = json.loads(custom)
-        concurrency.update(custom)
-
-    return concurrency
Author	SHA1	Message	Date
Nick Sweeting	d1136d74cb	fix docker release script	2024-12-15 02:06:49 -08:00
Nick Sweeting	54f598789e	bump more js and py dependency lockfiles	2024-12-15 00:33:03 -08:00
Nick Sweeting	5171eaf31e	bump dependencies and metadata for 0.7.3	2024-12-15 00:25:35 -08:00