Fix CLI tests and improve orchestrator performance

Key changes:
- Add DISABLE_ALL_PLUGINS config to allow disabling all plugins by default
- Fix NetworkInterface.current() exception handling in ArchiveResultMachine
- Add from_json/to_json aliases to Crawl, Snapshot, Tag, ArchiveResult models
- Fix CLI commands to use to_jsonl() for JSONL output (was using to_json() string)
- Fix migration 0025 to handle empty tables gracefully
- Fix migration 0027 uuid7 import from uuid_compat module
- Remove Process health stats fields (num_uses_failed/succeeded)
- Tests now pass with inline orchestrator mode for fast processing

.claude/settings.local.json

@@ -0,0 +1,31 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(python -m archivebox:*)",
+      "Bash(ls:*)",
+      "Bash(xargs:*)",
+      "Bash(python -c:*)",
+      "Bash(printf:*)",
+      "Bash(pkill:*)",
+      "Bash(python3:*)",
+      "Bash(sqlite3:*)",
+      "WebFetch(domain:github.com)",
+      "Bash(uv add:*)",
+      "Bash(mkdir:*)",
+      "Bash(chmod:*)",
+      "Bash(python -m forum_dl:*)",
+      "Bash(archivebox manage migrate:*)",
+      "Bash(cat:*)",
+      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
+      "Bash(forum-dl:*)",
+      "Bash(pip uninstall:*)",
+      "Bash(python:*)",
+      "Bash(source .venv/bin/activate)",
+      "Bash(mv:*)",
+      "Bash(echo:*)",
+      "Bash(grep:*)",
+      "WebFetch(domain:python-statemachine.readthedocs.io)",
+      "Bash(./bin/run_plugin_tests.sh:*)"
+    ]
+  }
+}

.dockerignore

@@ -9,16 +9,19 @@ __pycache__/
 .pdm-python
 .eggs/
 .git/
+.vscode/
 !.git/HEAD
 !.git/refs/heads/*
 
 venv/
 .venv/
 .venv-old/
+.docker_venv/
 .docker-venv/
 node_modules/
 chrome/
 chromeprofile/
+chrome_profile/
 
 pdm.dev.lock
 pdm.lock
@@ -31,7 +34,10 @@ deb_dist/
 pip_dist/
 assets/
 docker/
+website/
+typings/
 
+tmp/
 data/
 data*/
 output/

.gitattributes

@@ -1,2 +0,0 @@
-**/*.lock
-**/*-lock.json

.github/.readthedocs.yaml

@@ -0,0 +1,26 @@
+# Read the Docs config for https://docs.archivebox.io
+# https://docs.readthedocs.io/en/stable/config-file/v2.html
+
+version: 2
+
+submodules:
+  include: all
+  recursive: true
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    #nodejs: "20"     # not needed unless we need the full archivebox to run while building docs for some reason
+
+sphinx:
+  configuration: docs/conf.py
+
+formats:
+  - pdf
+  - epub
+
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt

.github/FUNDING.yml

@@ -1,2 +1,2 @@
 github: ["ArchiveBox", "pirate"]
-custom: ["https://donate.archivebox.io", "https://paypal.me/NicholasSweeting"]
+custom: ["https://donate.archivebox.io", "https://swag.archivebox.io"]

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -0,0 +1,198 @@
+name: 🐞 Bug report
+description: Report a bug or error you encountered in ArchiveBox
+title: "Bug: ..."
+assignees:
+  - pirate
+type: 'Bug'
+body:
+  - type: markdown
+    attributes:
+      value: |
+        *Please note:* it is normal to see errors occasionally for some extractors on some URLs (not every extractor will work on every type of page).  
+        Please report archiving errors if you are seeing them *consistently across many URLs* or if they are *preventing you from using ArchiveBox*.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Provide a screenshot and describe the bug
+      description: |
+        Attach a screenshot and describe what the issue is, what you expected to happen, and if relevant, the *URLs you were trying to archive*.  
+      placeholder: |
+        Got a bunch of 'singlefile was unable to archive this page' errors when trying to archive URLs from this site: https://example.com/xyz ...
+        I also tried to archive the same URLs using `singlefile` directly and some of them worked but not all of them. etc. ...
+    validations:
+      required: true
+
+  - type: textarea
+    id: steps_to_reproduce
+    attributes:
+      label: Steps to reproduce
+      description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
+      render: markdown
+      placeholder: |
+        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
+        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
+        3. Typed 'https://example.com/xyz' into the 'Add URL' input field
+        4. Clicked the 'Add+' button
+        5. Got a 500 error and saw the errors below in terminal
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs or errors
+      description: "Paste any terminal output, logs, or errors (check `data/logs/errors.log` as well)."
+      placeholder: |
+        ╭─────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+        │ [2024-11-02 19:54:28] ArchiveBox v0.8.6rc0: archivebox add https://example.com#1234567                  │
+        ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+        
+        [+] [2024-11-02 19:54:29] Adding 1 links to index (crawl depth=0)...
+            > Saved verbatim input to sources/1730577269-import.txt
+            > Parsed 1 URLs from input (Generic TXT)
+            ...
+      render: shell
+    validations:
+      required: false
+
+  - type: textarea
+    id: version
+    attributes:
+      label: ArchiveBox Version
+      description: |
+        **REQUIRED:** Run the `archivebox version` command inside your collection dir and paste the *full output* here (*not just the version number*).  
+        For Docker Compose run: `docker compose run archivebox version`  
+        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
+      render: shell
+      placeholder: |
+        0.8.6
+        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
+        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
+        
+        Binary Dependencies:
+        √  python     3.14.0     venv_pip   ~/.venv/bin/python
+        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
+        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
+        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
+        ...
+    validations:
+      required: true
+
+  - type: dropdown
+    id: install_method
+    validations:
+      required: true
+    attributes:
+      label: How did you install the version of ArchiveBox you are using?
+      multiple: false
+      options:
+        - pip
+        - apt
+        - brew
+        - nix
+        - Docker (or Podman/LXC/K8s/TrueNAS/Proxmox/etc)
+        - Other
+
+  - type: dropdown
+    id: operating_system
+    validations:
+      required: true
+    attributes:
+      label: What operating system are you running on?
+      description: |
+        Please note we are *unable to provide support for Windows users* unless you are using [Docker on Windows](https://github.com/ArchiveBox/archivebox#:~:text=windows%20without%20docker).
+      multiple: false
+      options:
+        - Linux (Ubuntu/Debian/Arch/Alpine/etc.)
+        - macOS (including Docker on macOS)
+        - BSD (FreeBSD/OpenBSD/NetBSD/etc.)
+        - Windows (including WSL, WSL2, Docker Desktop on Windows)
+        - Other
+
+  - type: checkboxes
+    id: filesystem
+    attributes:
+      label: What type of drive are you using to store your ArchiveBox data?
+      description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` OR `data/archive`?
+      options:
+        - label: "some of `data/` is on a local SSD or NVMe drive"
+          required: false
+        - label: "some of `data/` is on a spinning hard drive or external USB drive"
+          required: false
+        - label: "some of `data/` is on a network mount (e.g. NFS/SMB/Ceph/GlusterFS/etc.)"
+          required: false
+        - label: "some of `data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/Google Drive/Dropbox/etc.)"
+          required: false
+  
+
+  - type: textarea
+    id: docker_compose_yml
+    attributes:
+      label: Docker Compose Configuration
+      description: "If using Docker Compose, please share your full `docker-compose.yml` file. If using plain Docker, paste the `docker run ...` command you use."
+      placeholder: |
+        services:
+            archivebox:
+                image: archivebox/archivebox:latest
+                ports:
+                    - 8000:8000
+                volumes:
+                    - ./data:/data
+                environment:
+                    - ADMIN_USERNAME=admin
+                    - ADMIN_PASSWORD=****<redact any passwords>****
+                    - ALLOWED_HOSTS=*
+                    - CSRF_TRUSTED_ORIGINS=https://archivebox.example.com
+                    - PUBLIC_INDEX=True
+                    - PUBLIC_SNAPSHOTS=True
+                    - PUBLIC_ADD_VIEW=False
+                    ...
+
+            archivebox_scheduler:
+                image: archivebox/archivebox:latest
+                command: schedule --foreground --update --every=day
+                environment:
+                    ...
+      
+          ...
+      render: shell
+    validations:
+      required: false
+
+  - type: textarea
+    id: configuration
+    attributes:
+      label: ArchiveBox Configuration
+      description: "Please share your full `data/ArchiveBox.conf` file here."
+      render: shell
+      placeholder: |
+        [SERVER_CONFIG]
+        SECRET_KEY = "*********<redact any secrets/passwords>************"
+
+        WGET_RESTRICT_FILE_NAMES=windows
+        USE_SYSTEM_WGET=true
+        CHECK_SSL_VALIDITY=false
+        ...
+    validations:
+      required: false
+
+
+  - type: markdown
+    attributes:
+      value: |
+        ---
+          
+        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
+        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying anything suggested here, **we are not responsible for data loss**.  
+          
+        In the meantime please consider:  
+          
+          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
+          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
+          - 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
+          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
+        

.github/ISSUE_TEMPLATE/2-feature_request.yml

@@ -0,0 +1,128 @@
+name: 💡 Feature or enhancement request
+description: Suggest an idea or improvement for this project
+title: "Feature Request: ..."
+assignees:
+  - pirate
+type: 'Enhancement'
+labels: 'status: idea phase'
+body:
+  - type: dropdown
+    id: suggestion_type
+    validations:
+      required: true
+    attributes:
+      label: "What type of suggestion are you making?"
+      multiple: false
+      options:
+        - New extractor / type of content to save
+        - Proposing a new feature
+        - Modification of existing behavior
+        - Web UI or UX design improvement
+
+  - type: textarea
+    id: current_problem
+    attributes:
+      label: "What is the problem that your feature request solves?"
+      description: |
+        Describe the problem or need that your feature request solves, feel free to include any screenshots or examples.
+      placeholder: |
+        e.g. I need to be able to archive spanish and french subtitle files from a particular movie site https://example.com/somevideos that's going down soon.
+    validations:
+      required: true
+
+  - type: textarea
+    id: proposed_solution
+    attributes:
+      label: "What is your proposed solution?"
+      description: |
+        Describe the ideal specific solution you'd want, *and whether it fits into any broader scope of changes*.
+      placeholder: |
+        e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
+        The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
+    validations:
+      required: true
+
+  - type: textarea
+    id: workarounds_tried
+    attributes:
+      label: "What hacks or alternative solutions have you tried to solve the problem?"
+      description: |
+        A description of any alternative approaches, workarounds, or other solutions you've considered to fix the problem.
+      placeholder: |
+        e.g. I wait for archivebox to finish archiving the page, then I manually run `yt-dlp --subs <URL>` inside 
+        the `data/archive/<timestamp>/` directory to download the subtitle files and add them to the snapshot folder.
+    validations:
+      required: true
+
+  - type: textarea
+    id: version
+    attributes:
+      label: Share the entire output of the `archivebox version` command for the current verison you are using.
+      description: |
+        DO NOT JUST ENTER "the latest verion" OR YOUR ISSUE WILL BE CLOSED.  
+        We need to know what version of ArchiveBox and what feature flags you're currently running with in order to contextualize your feature request.  
+        Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in your specific environment.
+          
+        Run the `archivebox version` command inside your current collection dir and paste the *full output* here (*not just the version number*).  
+        For Docker Compose run: `docker compose run archivebox version`  
+        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
+      render: shell
+      placeholder: |
+        0.8.6
+        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
+        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
+        
+        Binary Dependencies:
+        √  python     3.14.0     venv_pip   ~/.venv/bin/python
+        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
+        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
+        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
+        ...
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: priority
+    attributes:
+      label: "How badly do you want this new feature?"
+      options:
+        - label: "It's an urgent deal-breaker, I can't live without it"
+          required: false
+        - label: "It's important to add it in the near-mid term future"
+          required: false
+        - label: "It would be nice to have eventually"
+          required: false
+        - label: "I'm willing to [start a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
+          required: false
+        - label: "I have [donated money](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations) to go towards fixing this issue"
+          required: false
+  
+  - type: checkboxes
+    id: satisfaction_survey
+    attributes:
+      label: Mini Survey
+      description: How do you like ArchiveBox so far?
+      options:
+        - label: "I like ArchiveBox so far / would recommend it to a friend"
+          required: false
+        - label: "I've had a lot of difficulty getting ArchiveBox set up"
+          required: false
+        - label: "I would pay $10/mo for a hosted version of ArchiveBox if it had this feature"
+          required: false
+
+  - type: markdown
+    attributes:
+      value: |
+        ---
+          
+        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
+        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying any workarounds or BETAs suggested here, **we are not responsible for data loss**.  
+          
+        In the meantime please consider:  
+          
+          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 📊 [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
+          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
+          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)

.github/ISSUE_TEMPLATE/3-documentation_change.yml

@@ -0,0 +1,52 @@
+name: 📑 Documentation improvement
+description: Submit an idea or correction for the Wiki documentation
+title: "Documentation: ..."
+labels: 'touches: docs'
+type: 'Enhancement'
+assignees:
+  - pirate
+body:
+  - type: markdown
+    attributes:
+      value: |
+        If you prefer, you can submit a [Pull Request](https://github.com/ArchiveBox/docs) on https://github.com/ArchiveBox/docs to edit the docs directly instead.
+
+  - type: input
+    id: page_url
+    validations:
+      required: true
+    attributes:
+      label: "What is the URL of the page you'd like to see improved?"
+      placeholder: e.g. https://github.com/ArchiveBox/docs/wiki/Install
+
+  - type: input
+    id: section_title
+    validations:
+      required: true
+    attributes:
+      label: "What is the title of the relevant section?"
+      placeholder: e.g. Option B. Automatic Setup Script
+
+  - type: textarea
+    id: suggested_edit
+    attributes:
+      label: "What is the suggested edit?"
+      placeholder: |
+        e.g. Please document how to run the automatic setup script for ArchiveBox on TempleOS.  
+        Attach images, screenshots, code snippets, etc. anything you think would help.
+    validations:
+      required: true
+
+  - type: markdown
+    attributes:
+      value: |
+        ---
+          
+        We strive to address issues as quickly as possible, it usually takes us *about a ~week* to respond.  
+          
+        In the meantime please consider:  
+          
+          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
+          - 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
+          - ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,46 +0,0 @@
----
-name: 🐞 Bug report
-about: Create a report to help us improve
-title: 'Bug: ...'
-labels: 'bug'
-assignees: ''
-
----
-
-<!--
-Please fill out the following information, 
-feel free to delete sections if they're not applicable 
-or if long issue templates annoy you.
-(the only required section is the version information)
--->
-
-#### Describe the bug
-<!--
-A description of what the bug is, 
-what you expected to happen, 
-and any relevant context about issue.
--->
-
-#### Steps to reproduce
-<!--
-For example:
-1. Ran ArchiveBox with the following config '...'
-2. Saw this output during archiving '....'
-3. UI didn't show the thing I was expecting '....'
--->
-
-#### Screenshots or log output
-
-<!--
-If applicable, post any relevant screenshots or copy/pasted terminal output from ArchiveBox.
-If you're reporting a parsing / importing error, **you must paste a copy of your redacted import file here**.
--->
-
-#### ArchiveBox version
-
-<!-- Run the `archivebox version` command locally then copy paste the result here: -->
-```logs
-replace this line with the *full*, unshortened output of running `archivebox version`
-```
-<!-- Tickets without full version info will closed until it is provided,
-we need the full output here to help you solve your issue -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: ❓ Ask a question or start a discussion
+    url: https://github.com/ArchiveBox/ArchiveBox/discussions
+    about: "Ask a question, get support, or start a design discussion (to report a problem please use '🐞 Bug report' instead)"
+  - name: 💬 Chat with the dev team & community on Zulip
+    url: https://zulip.archivebox.io
+    about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
+  - name: 💁‍♂️ Hire us for professional support with fast response times
+    url: https://docs.monadical.com/s/archivebox-consulting-services
+    about: "We provide hosting, develoment, and support, including on-prem/cloud w/ SSO & storage, CAPTCHA-solving, proxies, etc."

.github/ISSUE_TEMPLATE/documentation_change.md

@@ -1,17 +0,0 @@
----
-name: 📑 Documentation change
-about: Submit a suggestion for the Wiki documentation
-title: 'Documentation: Improvement request ...'
-labels: ''
-assignees: ''
-
----
-<!-- If you perfer, you can make a PR to https://github.com/ArchiveBox/docs instead of opening an issue -->
-
-## Wiki Page URL
-<!-- e.g. https://github.com/pirate/ArchiveBox/wiki/Configuration#use_color -->
-
-
-## Suggested Edit
-<!-- e.g. Please add more example usages, or please fix `xyz` typo to be `abc`. -->
-

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,50 +0,0 @@
----
-name: 💡 Feature request
-about: Suggest an idea for this project
-title: 'Feature Request: ...'
-labels: 'changes: behavior,status: idea phase'
-assignees: ''
-
----
-
-<!--
-Please fill out the following information, 
-feel free to delete sections if they're not applicable 
-or if long issue templates annoy you :)
--->
-
-## Type 
-
- - [ ] General question or discussion
- - [ ] Propose a brand new feature
- - [ ] Request modification of existing behavior or design
-
-## What is the problem that your feature request solves
-<!--
-e.g. I need to be able to archive spanish and french subtitle files  
-from a particular <example.com> movie site that's going down soon.
--->
-
-## Describe the ideal specific solution you'd want, and whether it fits into any broader scope of changes
-<!--
-e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
-The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
--->
-
-## What hacks or alternative solutions have you tried to solve the problem?
-<!--
-A clear and concise description of any alternative solutions, 
-workarounds, or other software you've considered using to fix the problem.
--->
-
-## How badly do you want this new feature?
-
- - [ ] It's an urgent deal-breaker, I can't live without it
- - [ ] It's important to add it in the near-mid term future
- - [ ] It would be nice to have eventually
-
----
-
- - [ ] I'm willing to contribute [dev time](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) / [money](https://github.com/sponsors/pirate) to fix this issue
- - [ ] I like ArchiveBox so far / would recommend it to a friend
- - [ ] I've had a lot of difficulty getting ArchiveBox set up

.github/ISSUE_TEMPLATE/question_or_discussion.md

@@ -1,9 +0,0 @@
----
-name: 💬 Question, discussion, or support request
-about: Start a discussion or ask a question about ArchiveBox
-title: 'Question: ...'
-labels: ''
-assignees: ''
-
----
-

.github/workflows/claude.yml

@@ -0,0 +1,50 @@
+name: Claude Code
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  issues:
+    types: [opened, assigned]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  claude:
+    if: |
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
+      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write
+      id-token: write
+      actions: read # Required for Claude to read CI results on PRs
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@v1
+        with:
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+
+          # This is an optional setting that allows Claude to read CI results on PRs
+          additional_permissions: |
+            actions: read
+
+          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
+          # prompt: 'Update the pull request description to include a summary of changes.'
+
+          # Optional: Add claude_args to customize behavior and configuration
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
+          claude_args: '--allowed-tools Bash(gh pr:*)'
+

.github/workflows/docker.yml

@@ -18,9 +18,9 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
+        # with:
+        #   submodules: true
+        #   fetch-depth: 1
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -40,26 +40,34 @@ jobs:
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-buildx-
 
-      - name: Docker Login
+      - name: Login to Docker Hub
         uses: docker/login-action@v3
         if: github.event_name != 'pull_request'
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Collect Docker tags
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Collect Full Release Docker tags
         # https://github.com/docker/metadata-action
         id: docker_meta
         uses: docker/metadata-action@v5
+        if: github.event_name == 'workflow_dispatch'
         with:
-          images: archivebox/archivebox,nikisweeting/archivebox
+          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
           tags: |
               # :stable
               type=ref,event=branch
@@ -71,6 +79,19 @@ jobs:
               type=sha
               # :latest
               type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
+      
+      - name: Collect Non-Release Docker tags
+        # https://github.com/docker/metadata-action
+        id: docker_meta_non_release
+        uses: docker/metadata-action@v5
+        if: github.event_name != 'workflow_dispatch'
+        with:
+          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
+          tags: |
+              # :stable
+              type=ref,event=branch
+              # :sha-463ea54
+              type=sha
 
       - name: Build and push
         id: docker_build
@@ -80,7 +101,8 @@ jobs:
           file: ./Dockerfile
           builder: ${{ steps.buildx.outputs.name }}
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.docker_meta.outputs.tags }}
+          tags: ${{ github.event_name == 'workflow_dispatch' ? steps.docker_meta.outputs.tags : steps.docker_meta_non_release.outputs.tags }}
+          labels: ${{ github.event_name == 'workflow_dispatch' ? steps.docker_meta.outputs.labels : steps.docker_meta_non_release.outputs.labels }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
           platforms: linux/amd64,linux/arm64

.github/workflows/duplicate-issue-detection.yml

@@ -0,0 +1,59 @@
+name: Duplicate Issue Detection
+
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  check-duplicates:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Install opencode
+        run: curl -fsSL https://opencode.ai/install | bash
+
+      - name: Check for duplicate issues
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENCODE_PERMISSION: |
+            {
+              "bash": {
+                "gh issue*": "allow",
+                "gh pr*": "allow",
+                "*": "deny"
+              },
+              "webfetch": "allow"
+            }
+        run: |
+          opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created:
+
+          Issue number:
+          ${{ github.event.issue.number }}
+
+          Lookup this issue and search through existing issues and PRs (excluding #${{ github.event.issue.number }}) in this repository to find any potential duplicates of this new issue.
+          Consider:
+          1. Similar titles or descriptions
+          2. Same error messages or symptoms
+          3. Related functionality or components
+          4. Similar feature requests
+
+          If you find any potential duplicates, please comment on the new issue with:
+          - A brief explanation of why it might be a duplicate
+          - Links to the potentially duplicate issues or PRs
+          - A suggestion to check those issues first
+
+          Use this format for the comment:
+          'This issue might be a duplicate of existing issues. Please check:
+          - #[issue_number]: [brief description of similarity]
+
+          Feel free to ignore if none of these address your specific case.'
+
+          If no clear duplicates are found, do not comment."

.github/workflows/gh-pages.yml

@@ -0,0 +1,64 @@
+# Simple workflow for deploying static content to GitHub Pages
+name: Deploy static content to Pages
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["dev"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Single deploy job since we're just deploying
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Copy README.md into place
+        run: |
+          rm -f ./website/README.md
+          cp ./README.md ./website/README.md
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+
+      # - name: Checkout
+      #   uses: actions/checkout@v4
+      # - name: Setup Pages
+      #   uses: actions/configure-pages@v5
+      # - name: Upload artifact
+      #   uses: actions/upload-pages-artifact@v3
+      #   with:
+      #     # Upload entire repository
+      #     path: './website'
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/pip.yml

@@ -3,76 +3,63 @@ name: Build Pip package
 on:
   workflow_dispatch:
   push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'
 
+env:
+  PYTHON_VERSION: "3.13"
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
-
     permissions:
       id-token: write
 
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
         with:
-          submodules: true
-          fetch-depth: 1
+          enable-cache: true
 
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: ${{ env.PYTHON_VERSION }}
           architecture: x64
 
-      - uses: awalsh128/cache-apt-pkgs-action@latest
+      - name: APT install archivebox dev + run dependencies
+        uses: awalsh128/cache-apt-pkgs-action@latest
         with:
           packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
           version: 1.0
 
-      - uses: pdm-project/setup-pdm@v3
+      - name: UV install archivebox dev + run sub-dependencies
+        run: uv sync --frozen --all-extras --no-install-project --no-install-workspace
+
+      - name: UV build archivebox and archivebox/pkgs/* packages
+        run: |
+          uv build --all
+
+      - name: Publish new package wheels and sdists to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # ignore when publish to PyPI fails due to duplicate tag
+        continue-on-error: true
         with:
-          python-version: '3.11'
-          cache: true
+          password: ${{ secrets.PYPI_PAT_SECRET }}
 
-      - name: Install dependencies
-        run: pdm install --fail-fast --no-lock --dev --group=':all' --no-self
+      - name: UV install archivebox and archivebox/pkgs/* locally for tests
+        run: uv sync --frozen --all-extras
 
-      - name: Build package
+      - name: UV run archivebox init + archivebox version
         run: |
-          pdm build
-
-      - name: Install from build
-        run: pip install ./dist/archivebox-*.whl
-
-      - name: Add some links to test
-        run: |
-          mkdir data && cd data
-          archivebox init
-          archivebox add 'https://example.com'
-          archivebox version
-          archivebox status
-
-      #- name: Publish package distributions to PyPI
-      #  run: pdm publish --no-build
-
-      # - name: Push build to PyPI
-      #   run: |
-      #     cd pip_dist/
-      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
-      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
-
-      # - name: Commit built package
-      #   run: |
-      #     cd pip_dist/
-      #     git config --local user.email "action@github.com"
-      #     git config --local user.name "GitHub Action"
-      #     git commit -m "Pip package autobuild" -a
-      
-      # - name: Push build to Github
-      #   uses: ad-m/github-push-action@master
-      #   with:
-      #     github_token: ${{ secrets.GITHUB_TOKEN }}
-      #     repository: ArchiveBox/pip-archivebox
-      #     branch: ${{ github.ref }}
-      #     directory: pip_dist
+          mkdir -p data && cd data
+          uv run archivebox init \
+          && uv run archivebox version
+          # && uv run archivebox add 'https://example.com' \
+          # && uv run archivebox status \
+          # || (echo "UV Failed to run archivebox!" && exit 1)
 

.github/workflows/test-parallel.yml

@@ -0,0 +1,118 @@
+name: Parallel Tests
+
+on:
+  pull_request:
+    branches: [dev, main, master]
+  push:
+    branches: [dev]
+
+env:
+  PYTHONIOENCODING: utf-8
+  PYTHONLEGACYWINDOWSSTDIO: utf-8
+  USE_COLOR: False
+
+jobs:
+  discover-tests:
+    name: Discover test files
+    runs-on: ubuntu-22.04
+    outputs:
+      test-files: ${{ steps.set-matrix.outputs.test-files }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Discover test files
+        id: set-matrix
+        run: |
+          # Find all main test files
+          main_tests=$(find tests -maxdepth 1 -name "test_*.py" -type f | sort)
+
+          # Find all plugin test files
+          plugin_tests=$(find archivebox/plugins -path "*/tests/test_*.py" -type f | sort)
+
+          # Combine and format as JSON array
+          all_tests=$(echo "$main_tests $plugin_tests" | tr ' ' '\n' | grep -v '^$')
+
+          # Create JSON array with test file info
+          json_array="["
+          first=true
+          for test_file in $all_tests; do
+            if [ "$first" = true ]; then
+              first=false
+            else
+              json_array+=","
+            fi
+
+            # Extract a display name for the test
+            if [[ $test_file == tests/* ]]; then
+              name="main/$(basename $test_file .py | sed 's/^test_//')"
+            else
+              plugin=$(echo $test_file | sed 's|archivebox/plugins/\([^/]*\)/.*|\1|')
+              test_name=$(basename $test_file .py | sed 's/^test_//')
+              name="plugin/$plugin/$test_name"
+            fi
+
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+          done
+          json_array+="]"
+
+          echo "test-files=$json_array" >> $GITHUB_OUTPUT
+          echo "Found $(echo $all_tests | wc -w) test files"
+          echo "$json_array" | jq '.'
+
+  run-tests:
+    name: ${{ matrix.test.name }}
+    runs-on: ubuntu-22.04
+    needs: discover-tests
+
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
+        python: ["3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Cache uv
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python }}-uv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.1
+
+      - name: Install dependencies with uv
+        run: |
+          uv sync --dev --all-extras
+
+      - name: Run test - ${{ matrix.test.name }}
+        run: |
+          uv run pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         os: [ubuntu-22.04]
         # os: [ubuntu-22.04, macos-latest, windows-latest]
-        python: [3.11]
+        python: ["3.13"]
 
     steps:
       - uses: actions/checkout@v4
@@ -38,7 +38,7 @@ jobs:
       - name: Setup PDM
         uses: pdm-project/setup-pdm@v3
         with:
-          python-version: '3.11'
+          python-version: '3.13'
           cache: true
 
       ### Install Python & JS Dependencies
@@ -102,7 +102,7 @@ jobs:
         # TODO: remove this exception for windows once we get tests passing on that platform
         if: ${{ !contains(matrix.os, 'windows') }}
         run: |
-          python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist
+          python -m pytest -s --basetemp=tests/out --ignore=archivebox/pkgs
 
   docker_tests:
     runs-on: ubuntu-latest

.gitignore

@@ -11,12 +11,14 @@ venv/
 .venv/
 .docker-venv/
 node_modules/
+typings/
 
 # Ignore dev lockfiles (should always be built fresh)
 pdm.dev.lock
 requirements-dev.txt
 
 # Packaging artifacts
+requirements.txt
 .pdm-python
 .pdm-build
 archivebox.egg-info
@@ -25,12 +27,16 @@ build/
 dist/
 
 # Data folders
+lib/
+tmp/
 data/
 data*/
 output/
 index.sqlite3
+queue.sqlite3
 *.sqlite*
 data.*
+.archivebox_id
 
 # vim
 *.sw?

.gitmodules

@@ -1,31 +1,3 @@
 [submodule "docs"]
     path = docs
     url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
-
-[submodule "deb_dist"]
-    path = deb_dist
-    url = https://github.com/ArchiveBox/debian-archivebox.git
-[submodule "brew_dist"]
-	path = brew_dist
-	url = https://github.com/ArchiveBox/homebrew-archivebox.git
-[submodule "pip_dist"]
-	path = pip_dist
-	url = https://github.com/ArchiveBox/pip-archivebox.git
-[submodule "docker"]
-	path = docker
-	url = https://github.com/ArchiveBox/docker-archivebox.git
-[submodule "archivebox/vendor/base32-crockford"]
-	path = archivebox/vendor/base32-crockford
-	url = https://github.com/jbittel/base32-crockford
-[submodule "archivebox/vendor/pocket"]
-	path = archivebox/vendor/pocket
-	url = https://github.com/tapanpandita/pocket
-[submodule "archivebox/vendor/django-taggit"]
-	path = archivebox/vendor/django-taggit
-	url = https://github.com/jazzband/django-taggit
-[submodule "archivebox/vendor/python-atomicwrites"]
-	path = archivebox/vendor/python-atomicwrites
-	url = https://github.com/untitaker/python-atomicwrites
-[submodule "archivebox/vendor/pydantic-pkgr"]
-	path = archivebox/vendor/pydantic-pkgr
-	url = https://github.com/ArchiveBox/pydantic-pkgr

.npmignore

@@ -1,19 +0,0 @@
-tests/
-archivebox/
-archivebox.egg-info/
-build/
-dist/
-docs/
-etc/
-.github
-.gitmodules
-.dockerignore
-.flake8
-CNAME
-_config.yml
-docker-compose.yaml
-docker-compose.yml
-Dockerfile
-MANIFEST.in
-Pipfile
-setup.py

.readthedocs.yaml

@@ -1,33 +0,0 @@
-# Read the Docs configuration file for Sphinx projects
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-version: 2
-
-submodules:
-  include: all
-  recursive: true
-
-# Set the OS, Python version and other tools you might need
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.12"
-    nodejs: "20"
-
-# Build documentation in the "docs/" directory with Sphinx
-sphinx:
-  configuration: docs/conf.py
-  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
-  # builder: "dirhtml"
-
-# Optionally build your docs in additional formats such as PDF and ePub
-formats:
-  - pdf
-  - epub
-
-# Optional but recommended, declare the Python requirements required
-# to build your documentation
-# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-python:
-  install:
-    - requirements: docs/requirements.txt

ArchiveBox.conf

@@ -0,0 +1,3 @@
+[SERVER_CONFIG]
+SECRET_KEY = y6fw9wcaqls9sx_dze6ahky9ggpkpzoaw5g5v98_u3ro5j0_4f
+

CLAUDE.md

@@ -0,0 +1,230 @@
+# Claude Code Development Guide for ArchiveBox
+
+## Quick Start
+
+```bash
+# Set up dev environment (always use uv, never pip directly)
+uv sync --dev --all-extras
+
+# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
+```
+
+## Development Environment Setup
+
+### Prerequisites
+- Python 3.11+ (3.13 recommended)
+- uv package manager
+- A non-root user for running tests (e.g., `testuser`)
+
+### Install Dependencies
+```bash
+uv sync --dev --all-extras  # Always use uv, never pip directly
+```
+
+### Activate Virtual Environment
+```bash
+source .venv/bin/activate
+```
+
+### Generate and Apply Migrations
+```bash
+# Generate migrations (run from archivebox subdirectory)
+cd archivebox
+./manage.py makemigrations
+
+# Apply migrations to test database
+cd data/
+archivebox init
+```
+
+## Running Tests
+
+### CRITICAL: Never Run as Root
+ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
+
+```bash
+# Run all migration tests
+sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
+
+# Run specific test file
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
+
+# Run single test
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
+```
+
+### Test File Structure
+```
+archivebox/tests/
+├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
+├── test_migrations_fresh.py      # Fresh install tests
+├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
+├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
+└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
+```
+
+## Test Writing Standards
+
+### NO MOCKS - Real Tests Only
+Tests must exercise real code paths:
+- Create real SQLite databases with version-specific schemas
+- Seed with realistic test data
+- Run actual `python -m archivebox` commands via subprocess
+- Query SQLite directly to verify results
+
+**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
+
+### NO SKIPS
+Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
+
+### Strict Assertions
+- `init` command must return exit code 0 (not `[0, 1]`)
+- Verify ALL data is preserved, not just "at least one"
+- Use exact counts (`==`) not loose bounds (`>=`)
+
+### Example Test Pattern
+```python
+def test_migration_preserves_snapshots(self):
+    """Migration should preserve all snapshots."""
+    result = run_archivebox(self.work_dir, ['init'], timeout=45)
+    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
+
+    ok, msg = verify_snapshot_count(self.db_path, expected_count)
+    self.assertTrue(ok, msg)
+```
+
+## Migration Testing
+
+### Schema Versions
+- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
+- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
+- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
+- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
+
+### Testing a Migration Path
+1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
+2. Seed with realistic test data using `seed_0_X_data()`
+3. Run `archivebox init` to trigger migrations
+4. Verify data preservation with `verify_*` functions
+5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
+
+### Squashed Migrations
+When testing 0.8.x (dev branch), you must record ALL replaced migrations:
+```python
+# The squashed migration replaces these - all must be recorded
+('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
+('core', '0024_auto_20240513_1143'),
+# ... all 52 migrations from 0023-0074 ...
+('core', '0023_new_schema'),  # Also record the squashed migration itself
+```
+
+## Common Gotchas
+
+### 1. File Permissions
+New files created by root need permissions fixed for testuser:
+```bash
+chmod 644 archivebox/tests/test_*.py
+```
+
+### 2. DATA_DIR Environment Variable
+ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
+
+### 3. Extractors Disabled for Speed
+Tests disable all extractors via environment variables for faster execution:
+```python
+env['SAVE_TITLE'] = 'False'
+env['SAVE_FAVICON'] = 'False'
+# ... etc
+```
+
+### 4. Timeout Settings
+Use appropriate timeouts for migration tests (45s for init, 60s default).
+
+### 5. Circular FK References in Schemas
+SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
+
+## Architecture Notes
+
+### Crawl Model (0.9.x)
+- Crawl groups multiple Snapshots from a single `add` command
+- Each `add` creates one Crawl with one or more Snapshots
+- Seed model was removed - crawls now store URLs directly
+
+### Migration Strategy
+- Squashed migrations for clean installs
+- Individual migrations recorded for upgrades from dev branch
+- `replaces` attribute in squashed migrations lists what they replace
+
+## Code Style Guidelines
+
+### Naming Conventions for Grep-ability
+Use consistent naming for everything to enable easy grep-ability and logical grouping:
+
+**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
+
+**Examples**:
+```python
+# Filesystem migration methods - all start with fs_
+def fs_migration_needed() -> bool: ...
+def fs_migrate() -> None: ...
+def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
+def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
+def _fs_next_version(current: str) -> str: ...
+
+# Logging methods - ALL must start with log_ or _log
+def log_migration_start(snapshot_id: str) -> None: ...
+def _log_error(message: str) -> None: ...
+def log_validation_result(ok: bool, msg: str) -> None: ...
+```
+
+**Rules**:
+- Group related functions with common prefixes
+- Use `_` prefix for internal/private helpers within the same family
+- ALL logging-related methods MUST start with `log_` or `_log`
+- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
+- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
+
+### Minimize Unique Names and Data Structures
+**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
+
+**Example - GOOD**:
+```python
+# Binary has overrides field
+binary = Binary(overrides={'TIMEOUT': '60s'})
+
+# Binary reuses the same field name and structure
+class Binary(models.Model):
+    overrides = models.JSONField(default=dict)  # Same name, same structure
+```
+
+**Example - BAD**:
+```python
+# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
+class Binary(models.Model):
+    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
+```
+
+**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
+
+## Debugging Tips
+
+### Check Migration State
+```bash
+sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
+```
+
+### Check Table Schema
+```bash
+sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
+```
+
+### Verbose Test Output
+```bash
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
+```
+
+### Kill Zombie Chrome Processes
+```bash
+./bin/kill_chrome.sh
+```

Dockerfile

@@ -1,22 +1,34 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
-#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file
+# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
+#     python3.14, pip, pipx, uv, python3-ldap
+#     curl, wget, git, dig, ping, tree, nano
+#     node, npm, single-file, readability-extractor, postlight-parser
+#     ArchiveBox, yt-dlp, playwright, chromium
 # Usage:
-#     git submodule update --init --recursive
-#     git pull --recurse-submodules
-#     docker build . -t archivebox --no-cache
+#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
+#     docker build . -t archivebox
 #     docker run -v "$PWD/data":/data archivebox init
 #     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
 #     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
 # Multi-arch build:
 #     docker buildx create --use
-#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
-#
-# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
+#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
+# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
 
 
-# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
-FROM python:3.11-slim-bookworm
+#########################################################################################
+
+### Example: Using ArchiveBox in your own project's Dockerfile ########
+
+# FROM python:3.14-slim
+# WORKDIR /data
+# RUN pip install archivebox>=0.8.5rc51   # use latest release here
+# RUN archivebox install
+# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
+
+#########################################################################################
+
+FROM ubuntu:24.04
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -42,10 +54,9 @@ ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
-
 ######### Environment Variables #################################
 
-# Global system-level config
+# Global built-time and runtime environment constants + default pkg manager config
 ENV TZ=UTC \
     LANGUAGE=en_US:en \
     LC_ALL=C.UTF-8 \
@@ -57,47 +68,43 @@ ENV TZ=UTC \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     npm_config_loglevel=error
 
-# Version config
-ENV PYTHON_VERSION=3.11 \
-    NODE_VERSION=20
+# Language Version config
+ENV PYTHON_VERSION=3.12 \
+    NODE_VERSION=22
 
-# User config
+# Non-root User config
 ENV ARCHIVEBOX_USER="archivebox" \
     DEFAULT_PUID=911 \
-    DEFAULT_PGID=911
+    DEFAULT_PGID=911 \
+    IN_DOCKER=True
 
-# Global paths
+# ArchiveBox Source Code + Lib + Data paths
 ENV CODE_DIR=/app \
     DATA_DIR=/data \
-    GLOBAL_VENV=/venv \
     PLAYWRIGHT_BROWSERS_PATH=/browsers
+    # GLOBAL_VENV=/venv \
+    # TODO: add TMP_DIR and LIB_DIR?
 
-# Application-level paths
-ENV APP_VENV=/app/.venv \
-    NODE_MODULES=/app/node_modules
-
-# Build shell config
-ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
+# Bash SHELL config
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
 
 ######### System Environment ####################################
 
-# Detect ArchiveBox version number by reading package.json
-COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
-RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt
+# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
+WORKDIR "$CODE_DIR"
 
-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
+# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
 RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
     && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
     && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
     && rm -f /etc/apt/apt.conf.d/docker-clean
 
 # Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
+RUN (echo "[i] Docker build for ArchiveBox starting..." \
     && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
     && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
     && echo \
-    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
     && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
     && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
     && echo \
@@ -121,30 +128,59 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
 
 # Install system apt dependencies (adding backports to access more recent apt updates)
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
-    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' > /etc/apt/sources.list.d/backports.list \
+    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
     && mkdir -p /etc/apt/keyrings \
     && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
+    && apt-get install -qq -y \
         # 1. packaging dependencies
         apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
         # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep \
+        zlib1g-dev dumb-init gosu cron unzip grep dnsutils \
         # 3. frivolous CLI helpers to make debugging failed archiving easier
+        tree nano iputils-ping \
         # nano iputils-ping dnsutils htop procps jq yq
     && rm -rf /var/lib/apt/lists/*
 
+# Install apt binary dependencies for exractors
+# COPY --from=selenium/ffmpeg:latest /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing extractor dependencies for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        git ripgrep \
+        # Packages we have also needed in the past:
+        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
+        # curl wget (already installed above)
+    && rm -rf /var/lib/apt/lists/* \
+    # Save version info
+    && ( \
+        which curl && curl --version | head -n1 \
+        && which wget && wget --version 2>&1 | head -n1 \
+        && which git && git --version 2>&1 | head -n1 \
+        # && which ffmpeg && (ffmpeg --version 2>&1 | head -n1) || true \
+        && which rg && rg --version 2>&1 | head -n1 \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install sonic search backend
+COPY --from=archivebox/sonic:1.4.9 /usr/local/bin/sonic /usr/local/bin/sonic
+COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
+RUN (which sonic && sonic --version) | tee -a /VERSION.txt
+
 ######### Language Environments ####################################
 
-# Install Python environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
+# Set up Python environment
+# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
+#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
     # && apt-get update -qq \
-    # && apt-get install -qq -y -t bookworm-backports --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip \
+    # && apt-get install -qq -y --no-upgrade \
+    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
     # && rm -rf /var/lib/apt/lists/* \
     # tell PDM to allow using global system python site packages
     # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
+    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
     # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
     # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
     # && python3 -m venv --system-site-packages $GLOBAL_VENV \
@@ -152,22 +188,23 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     # install global dependencies / python build dependencies in GLOBAL_VENV
     # && pip install --upgrade pip setuptools wheel \
     # Save version info
-    && ( \
-        which python3 && python3 --version | grep " $PYTHON_VERSION" \
-        && which pip && pip --version \
-        # && which pdm && pdm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
+    # && ( \
+    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
+    #     && which pip && pip --version \
+    #     # && which pdm && pdm --version \
+    #     && echo -e '\n\n' \
+    # ) | tee -a /VERSION.txt
 
 
-# Install Node environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
+# Set up Node environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing NODE $NODE_VERSION for $TARGETPLATFORM..." \
     && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
     && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
     && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-upgrade libatomic1 \
-    && apt-get install -y -t bookworm-backports --no-upgrade \
+    && apt-get install -qq -y --no-upgrade libatomic1 \
+    && apt-get install -y --no-upgrade \
         nodejs \
     && rm -rf /var/lib/apt/lists/* \
     # Update NPM to latest version
@@ -180,141 +217,177 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     ) | tee -a /VERSION.txt
 
 
-######### Extractor Dependencies ##################################
-
-# Install apt dependencies
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing APT extractor dependencies globally using apt..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        curl wget git yt-dlp ffmpeg ripgrep \
-        # Packages we have also needed in the past:
-        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
-    && rm -rf /var/lib/apt/lists/* \
-    # Save version info
+# Set up uv and main app /venv
+COPY --from=ghcr.io/astral-sh/uv:0.5 /uv /uvx /bin/
+ENV UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_PREFERENCE=only-system \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv
+WORKDIR "$CODE_DIR"
+# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM} (provided by base image)..." \
+    && uv python find --system \
+    && uv venv /venv
+ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
+RUN uv pip install setuptools pip \
     && ( \
-        which curl && curl --version | head -n1 \
-        && which wget && wget --version 2>&1 | head -n1 \
-        && which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
-        && which git && git --version 2>&1 | head -n1 \
-        && which rg && rg --version 2>&1 | head -n1 \
+        which python3 && python3 --version \
+        && which uv && uv version \
+        && uv python find --system && uv python find \
         && echo -e '\n\n' \
     ) | tee -a /VERSION.txt
 
-# Install chromium browser using playwright
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
+
+######### ArchiveBox & Extractor Dependencies ##################################
+
+# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
     && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-        at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
-        libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
-        libxaw7 libxcomposite1 libxdamage1 libxfont2 \
-        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils xfonts-encodings \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential gcc \
+        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
+        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
+    && uv pip install \
+        "python-ldap>=3.4.3" \
+    && apt-get purge -y \
+        python3-dev build-essential gcc \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+
+# Install apt font & rendering dependencies for chromium browser
+# TODO: figure out how much of this overlaps with `playwright install-deps chromium`
+# RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+
+# Install chromium browser binary using playwright
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing CHROMIUM dependencies, fonts, and display libraries for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y \
+        #fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+        #at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
+        #libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
+        #libxaw7 libxcomposite1 libxdamage1 libxfont2 \
+        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
         # xfonts-scalable xfonts-utils xserver-common xvfb \
         # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
         # libxss1 dbus dbus-x11 upower \
     # && service dbus start \
-    # install Chromium using playwright
-    && pip install playwright \
-    && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
-    && playwright install chromium \
-    && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
-    && rm -rf /var/lib/apt/lists/* \
+    && echo "[+] PIP Installing playwright into /venv and CHROMIUM binary into $PLAYWRIGHT_BROWSERS_PATH..." \
+    && uv pip install "playwright>=1.49.1" \
+    && uv run playwright install chromium --no-shell --with-deps \  
+    && export CHROME_BINARY="$(uv run python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
     && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
+    && ln -s /browsers/ffmpeg-*/ffmpeg-linux /usr/bin/ffmpeg \
     && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
-    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \
     && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
     && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
+    # delete extra full copy of node that playwright installs (saves >100mb)
+    && rm -f /venv/lib/python$PYTHON_VERSION/site-packages/playwright/driver/node \
     # Save version info
+    && rm -rf /var/lib/apt/lists/* \
     && ( \
-        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
+        uv pip show playwright \
+        && which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
+        && which ffmpeg && ffmpeg -version \
         && echo -e '\n\n' \
     ) | tee -a /VERSION.txt
 
-# Install Node dependencies
+# Install Node extractor dependencies
+ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH"
+USER $ARCHIVEBOX_USER
+WORKDIR "/home/$ARCHIVEBOX_USER/.npm"
+RUN --mount=type=cache,target=/home/archivebox/.npm_cache,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT,uid=$DEFAULT_PUID,gid=$DEFAULT_PGID \
+    echo "[+] NPM Installing node extractor dependencies into /home/$ARCHIVEBOX_USER/.npm..." \
+    && npm config set prefix "/home/$ARCHIVEBOX_USER/.npm" \
+    && npm install --global --prefer-offline --no-fund --no-audit --cache "/home/$ARCHIVEBOX_USER/.npm_cache" \
+        "@postlight/parser@^2.2.3" \
+        "readability-extractor@github:ArchiveBox/readability-extractor" \
+        "single-file-cli@^1.1.54" \
+        "puppeteer@^23.5.0" \
+        "@puppeteer/browsers@^2.4.0" \
+    && rm -Rf "/home/$ARCHIVEBOX_USER/.cache/puppeteer"
+USER root
 WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
-RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
-    && npm ci --prefer-offline --no-audit --cache /root/.npm \
-    && ( \
+RUN ( \
         which node && node --version \
         && which npm && npm version \
+        && which postlight-parser \
+        && which readability-extractor && readability-extractor --version \
+        && which single-file && single-file --version \
+        && which puppeteer && puppeteer --version \
         && echo -e '\n\n' \
     ) | tee -a /VERSION.txt
 
 ######### Build Dependencies ####################################
 
-# Install ArchiveBox Python dependencies
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        build-essential \
-        libssl-dev libldap2-dev libsasl2-dev \
-        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
-    # && pdm use --venv in-project \
-    # && pdm run python -m ensurepip \
-    # && pdm sync --fail-fast --no-editable --group :all --no-self \
-    # && pdm export -o requirements.txt --without-hashes \
-    # && source $GLOBAL_VENV/bin/activate \
-    && pip install -r requirements.txt \
-    && apt-get purge -y \
-        build-essential \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
 
-# Install ArchiveBox Python package from source
+# Install ArchiveBox Python venv dependencies from uv.lock
+RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml and uv.lock..." \
+    && uv sync \
+        --frozen \
+        --inexact \
+        --all-extras \
+        --no-install-project \
+        --no-install-workspace
+    # installs the pip packages that archivebox depends on, defined in pyproject.toml and uv.lock dependencies
+
+# Install ArchiveBox Python package + workspace dependencies from source
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
-    # && apt-get update -qq \
-    # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
-    # && apt-get install -qq -y -t bookworm-backports \
-        # build-essential  \
-    # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
-    && pip install -e "$CODE_DIR"[sonic,ldap] \
-    # save docker image size and always remove compilers / build tools after building is complete
-    # && apt-get purge -y build-essential \
-    # && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && uv sync \
+        --frozen \
+        --inexact \
+        --all-extras \
+    && ( \
+        uv tree \
+        && which archivebox \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
 
 ####################################################
 
 # Setup ArchiveBox runtime config
-WORKDIR "$DATA_DIR"
-ENV IN_DOCKER=True \
-    DISPLAY=novnc:0.0 \
-    CUSTOM_TEMPLATES_DIR=/data/templates \
+ENV TMP_DIR=/tmp/archivebox \
+    LIB_DIR=/usr/share/archivebox/lib \
     GOOGLE_API_KEY=no \
     GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no \
-    ALLOWED_HOSTS=*
-    ## No need to set explicitly, these values will be autodetected by archivebox in docker:
-    # WGET_BINARY="wget" \
-    # YOUTUBEDL_BINARY="yt-dlp" \
-    # CHROME_BINARY="/usr/bin/chromium-browser" \
-    # USE_SINGLEFILE=True \
-    # SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
-    # USE_READABILITY=True \
-    # READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
-    # USE_MERCURY=True \
-    # MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
+WORKDIR "$DATA_DIR"
+RUN openssl rand -hex 16 > /etc/machine-id \
+    && mkdir -p "$TMP_DIR" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
+    && mkdir -p "$LIB_DIR" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
+    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
 
 # Print version for nice docker finish summary
 RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
     && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
     && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
     ) | tee -a /VERSION.txt
+
+# Run   $ archivebox version                                >> /VERSION.txt
+# RUN "$CODE_DIR"/bin/docker_entrypoint.sh init 2>&1 | tee -a /VERSION.txt
 RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
 
 ####################################################
 
-# Open up the interfaces to the outside world
+# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Nick Sweeting
+Copyright (c) 2024 Nick Sweeting
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -13,23 +13,25 @@
 <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE"><img src="https://img.shields.io/badge/Open_source-MIT-green.svg?logo=git&logoColor=green"/></a> <a href="https://github.com/ArchiveBox/ArchiveBox/commits/dev"><img src="https://img.shields.io/github/last-commit/ArchiveBox/ArchiveBox.svg?logo=Sublime+Text&logoColor=green&label=Active"/></a> &nbsp; <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?logo=github&label=Stars&logoColor=blue"/></a> &nbsp; <a href="https://hub.docker.com/r/archivebox/archivebox"><img src="https://img.shields.io/docker/pulls/archivebox/archivebox.svg?label=Docker+Pulls"/></a> <a href="https://pypi.org/project/archivebox/"><img src="https://img.shields.io/pypi/dm/archivebox?label=PyPI%20Installs&color=%235f7dae"/></a> <a href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj"><img src="https://img.shields.io/chrome-web-store/users/habonpimjphpdnmcfkaockjnffodikoj?label=Chrome%20Store&color=%231973e8"/></a>
 
 <!--<pre lang="bash" align="left"><code style="white-space: pre-line; text-align: left" align="left">
-curl -fsSL 'https://get.archivebox.io' | sh    # (or see pip/brew/Docker instructions below)
+curl -fsSL 'https://get.archivebox.io' | bash    # (or see pip/brew/Docker instructions below)
 </code></pre>-->
 
 </div>
 <hr/>
 <br/>
 
-**ArchiveBox is a powerful, self-hosted internet archiving solution to collect, save, and view websites offline.**
+**ArchiveBox is a self-hosted app that lets you preserve content from websites in a variety of formats.**
 
-Without active preservation effort, everything on the internet eventually dissapears or degrades. Archive.org does a great job as a centralized service, but saved URLs have to be public, and they can't save every type of content.
+We aim to make your data immediately useful, and kept in formats that other programs can read directly. As output, we save standard HTML, PNG, PDF, TXT, JSON, WARC, SQLite, all guaranteed to be readable for decades to come. ArchiveBox also has a CLI, REST API, and webhooks so you can set up integrations with other services.
+
+Without active preservation effort, everything on the internet eventually disappears or degrades.
 
 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>
 
-> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart), [macOS](#quickstart), and [Windows](#quickstart) (WSL2), or via **[Docker](#quickstart)** ⭐️.  
+> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
 
-*Once installed, it can be used as a [CLI tool](#usage), [self-hosted Web App](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python library](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [one-off command](#static-archive-exporting).*
+*Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*
 
 <br/>
 <hr/>
@@ -99,7 +101,7 @@ archivebox init --setup
 <br/>
 <br/>
 # Option D: Or use the optional auto setup script to install it
-curl -fsSL 'https://get.archivebox.io' | sh
+curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
 <sub>Open <a href="http://localhost:8000"><code>http://localhost:8000</code></a> to see your server's Web UI ➡️</sub>
@@ -130,7 +132,7 @@ curl -fsSL 'https://get.archivebox.io' | sh
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
-- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
 - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
 - Advanced users: support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
@@ -152,7 +154,7 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 - **Governments:**
   `snapshoting public service sites`, `recordkeeping compliance`
 
-> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  (we are also seeking [grant funding](https://github.com/ArchiveBox/ArchiveBox/issues/1126#issuecomment-1487431394))  
+> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  
 > We offer: setup & support, CAPTCHA/ratelimit unblocking, SSO, audit logging/chain-of-custody, and more  
 > *ArchiveBox is a 🏛️ 501(c)(3) [nonprofit FSP](https://hackclub.com/hcb/) and all our work supports open-source development.* 
 
@@ -233,7 +235,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (optional, highly recommended but not required).</li>
 <li>Run the automatic setup script.
-<pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | sh</code></pre>
+<pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>
@@ -257,7 +259,7 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 <ol>
 
 <li>Install <a href="https://realpython.com/installing-python/">Python >= v3.10</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v18</a> on your system (if not already installed).</li>
-<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://pipx.pypa.io"><code>pipx</code></a>).
+<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://docs.astral.sh/uv/guides/tools/#running-tools"><code>uvx</code></a>).
 <pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox yt-dlp playwright
 playwright install --with-deps chromium
 archivebox version
@@ -368,7 +370,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 
 <ul>
 <li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>)</li>
-<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | sh</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
+<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
 <li>Nix: <a href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/archivebox/default.nix"><code>nix-env --install archivebox</code></a> (contributed by <a href="https://github.com/siraben"><code>@siraben</code></a>)</li>
 <li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
 <li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
@@ -409,6 +411,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <ul>
 <li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
 <li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
+<li><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a></li>
 <li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
 <li><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a></li>
 <li><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a></li>
@@ -498,7 +501,7 @@ docker run -it -v $PWD:/data archivebox/archivebox help
 
 - `archivebox` `help`/`version` to see the list of available subcommands / currently installed version info
 - `archivebox` `setup`/`init`/`config`/`status`/`shell`/`manage` to administer your collection
-- `archivebox` `add`/`oneshot`/`schedule` to pull in fresh URLs from [bookmarks/history/RSS/etc.](#input-formats)
+- `archivebox` `add`/`schedule` to pull in fresh URLs from [bookmarks/history/RSS/etc.](#input-formats)
 - `archivebox` `list`/`update`/`remove` to manage existing Snapshots in your collection
 
 <br/>
@@ -760,8 +763,8 @@ The configuration is documented here: **[Configuration Wiki](https://github.com/
 <br/>
 TIMEOUT=240                # default: 60    add more seconds on slower networks
 CHECK_SSL_VALIDITY=False   # default: True  False = allow saving URLs w/ bad SSL
-SAVE_ARCHIVE_DOT_ORG=False # default: True  False = disable Archive.org saving
-MAX_MEDIA_SIZE=1500m       # default: 750m  raise/lower youtubedl output size
+SAVE_ARCHIVEDOTORG=False # default: True  False = disable Archive.org saving
+YTDLP_MAX_SIZE=1500m       # default: 750m  raise/lower yt-dlp output size
 <br/>
 PUBLIC_INDEX=True          # default: True  whether anon users can view index
 PUBLIC_SNAPSHOTS=True      # default: True  whether anon users can view pages
@@ -790,7 +793,19 @@ ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.c
 <p><em>TIP: For better security while running ArchiveBox, and to avoid polluting your host system with a bunch of sub-dependencies that you need to keep up-to-date,<strong>it is strongly recommended to use the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">⭐️ official Docker image</a></strong> which provides everything in an easy container with simple one-liner upgrades.</em></p>
 </blockquote>
 
-These optional dependencies used for archiving sites include:
+<ul>
+<li>Language: Python <code>&gt;=3.10</code></li>
+<li>Backend: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://django-ninja.dev/">Django-Ninja</a> for REST API</li>
+<li>Frontend: <a href="https://docs.djangoproject.com/en/5.1/ref/contrib/admin/">Django Admin</a> + Vanilla HTML, CSS, JS</li>
+<li>Web Server: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://channels.readthedocs.io/en/latest/"><code>channels</code></a> + <a href="https://github.com/django/daphne/"><code>daphne]</code></a></li>
+<li>Database: <a href="https://docs.djangoproject.com/en/5.1/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite</code></li>
+<li>Job Queue: <a href="https://huey.readthedocs.io/">Huey</a> using <code>./data/queue.sqlite3</code> under <code>supervisord</code></li>
+<li>Build/test/lint: <a href="https://github.com/pdm-project/pdm"><code>pdm</code></a> / <code>mypy</code>+<code>pyright</code>+<code>pytest</code> / <code>ruff</code></li>
+<li>Subdependencies: <a href="https://github.com/ArchiveBox/abx-pkg"><code>abx-pkg</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
+</ul>
+
+
+These optional subdependencies used for archiving sites include:
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/62a02155-05d7-4f3e-8de5-75a50a145c4f" alt="archivebox --version CLI output screenshot showing dependencies installed" width="330px" align="right" style="max-width: 100%;">
 
@@ -885,7 +900,7 @@ Each snapshot subfolder <code>data/archive/TIMESTAMP/</code> includes a static <
 
 ## Static Archive Exporting
 
-You can create one-off archives of individual URLs with `archivebox oneshot`, or export your index as static HTML using `archivebox list` (so you can view it without an ArchiveBox server).
+You can export your index as static HTML using `archivebox list` (so you can view it without an ArchiveBox server).
 
 <br/>
 <details>
@@ -895,10 +910,7 @@ You can create one-off archives of individual URLs with `archivebox oneshot`, or
 <p><em>NOTE: These exports are not paginated, exporting many URLs or the entire archive at once may be slow. Use the filtering CLI flags on the <code>archivebox list</code> command to export specific Snapshots or ranges.</em></p>
 </blockquote>
 
-<pre lang="bash"><code style="white-space: pre-line"># do a one-off single URL archive wihout needing a data dir initialized
-archivebox oneshot 'https://example.com'
-
-# archivebox list --help
+<pre lang="bash"><code style="white-space: pre-line"># archivebox list --help
 archivebox list --html --with-headers > index.html     # export to static html table
 archivebox list --json --with-headers > index.json     # export to json blob
 archivebox list --csv=timestamp,url,title > index.csv  # export to csv spreadsheet
@@ -944,7 +956,7 @@ archivebox add 'https://docs.google.com/document/d/12345somePrivateDocument'
 archivebox add 'https://vimeo.com/somePrivateVideo'
 
 # without first disabling saving to Archive.org:
-archivebox config --set SAVE_ARCHIVE_DOT_ORG=False  # disable saving all URLs in Archive.org
+archivebox config --set SAVE_ARCHIVEDOTORG=False  # disable saving all URLs in Archive.org
 
 # restrict the main index, Snapshot content, and Add Page to authenticated users as-needed:
 archivebox config --set PUBLIC_INDEX=False
@@ -1077,7 +1089,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <br/>
 
 <ul>
-<li><strong>ArchiveBox can use anywhere from ~1gb per 1000 Snapshots, to ~50gb per 1000 Snapshots</strong>, mostly dependent on whether you're saving audio & video using <code>SAVE_MEDIA=True</code> and whether you lower <code>MEDIA_MAX_SIZE=750mb</code>.</li>
+<li><strong>ArchiveBox can use anywhere from ~1gb per 1000 Snapshots, to ~50gb per 1000 Snapshots</strong>, mostly dependent on whether you're saving video/audio using <code>YTDLP_ENABLED=True</code> and whether you lower <code>YTDLP_MAX_SIZE=750m</code>.</li>
 <li>Disk usage can be reduced by using a compressed/<a href="https://www.ixsystems.com/blog/ixsystems-and-klara-systems-celebrate-valentines-day-with-a-heartfelt-donation-of-fast-dedupe-to-openzfs-and-truenas/">deduplicated</a> filesystem like <a href="https://www.reddit.com/r/zfs/comments/t9cexx/a_simple_real_world_zfs_compression_speed_an/">ZFS</a>/BTRFS, or by turning off extractors methods you don't need. You can also deduplicate content with a tool like <a href="https://github.com/adrianlopezroche/fdupes"><code>fdupes</code></a> or <a href="https://github.com/pauldreik/rdfind"><code>rdfind</code></a>.  
 </li>
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
@@ -1221,7 +1233,7 @@ ArchiveBox differentiates itself from [similar self-hosted projects](https://git
 
 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*
 
-*If you want more bookmark categorization and note-taking features, check out [Archivy](https://archivy.github.io/), [Memex](https://github.com/WorldBrain/Memex), [Polar](https://getpolarized.io/), or [LinkAce](https://www.linkace.org/).*
+*If you want more bookmark categorization and note-taking features, check out [Memex](https://github.com/WorldBrain/Memex), [Hoarder](https://github.com/hoarder-app/hoarder), [LinkWarden](https://github.com/linkwarden/linkwarden),  [Archivy](https://archivy.github.io/), or [LinkAce](https://www.linkace.org/).*
 
 *If you need more advanced recursive spider/crawling ability beyond `--depth=1`, check out [Browsertrix](https://github.com/webrecorder/browsertrix-crawler), [Photon](https://github.com/s0md3v/Photon), or [Scrapy](https://scrapy.org/) and pipe the outputted URLs into ArchiveBox.*
 
@@ -1307,8 +1319,8 @@ We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wik
 ## Developers
 
 - [Developer Documentation](https://github.com/ArchiveBox/ArchiveBox#archivebox-development)
-- [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha)
-- [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha)
+- [Python API](https://docs.archivebox.io/)
+- [REST API](https://demo.archivebox.io/api) (alpha)
 
 ## More Info
 
@@ -1335,6 +1347,9 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc
 
 **Python API Documentation:** https://docs.archivebox.io/en/dev/archivebox.html#module-archivebox.main
 
+**Internal Architecture Diagrams:** https://github.com/ArchiveBox/ArchiveBox/wiki/ArchiveBox-Architecture-Diagrams
+
+
 ### Setup the dev environment
 
 <details><summary><i>Click to expand...</i></summary>
@@ -1353,18 +1368,18 @@ git pull --recurse-submodules
 
 ```bash
 # Install ArchiveBox + python dependencies
-python3 -m venv .venv && source .venv/bin/activate && pip install -e '.[dev]'
-# or: pipenv install --dev && pipenv shell
+pip install uv
+./bin/lock_pkgs.sh         # (aka `uv venv; uv sync;` + generate requirements.txt)
 
-# Install node dependencies
-npm install
-# or
-archivebox setup
+# Install ArchiveBox runtime dependencies
+mkdir -p data && cd data
+archivebox install         # on >=v0.8.5 (otherwise `archivebox setup`)
 
-# Check to see if anything is missing
-archivebox --version
-# install any missing dependencies manually, or use the helper script:
-./bin/setup.sh
+# Run the development server w/ autoreloading (but no bg workers)
+archivebox manage runserver --debug --reload 0.0.0.0:8000
+
+# Run the production server (with bg workers but no autoreloading)
+archivebox server 0.0.0.0:8000
 ```
 
 #### 2. Option B: Build the docker container and use that for development instead
@@ -1373,14 +1388,15 @@ archivebox --version
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
 # inside the container will reload and pick up your changes
-docker build . -t archivebox
-docker run -it \
-    -v $PWD/data:/data \
-    archivebox init --setup
-docker run -it -p 8000:8000 \
-    -v $PWD/data:/data \
-    -v $PWD/archivebox:/app/archivebox \
-    archivebox server 0.0.0.0:8000 --debug --reload
+./bin/build_docker.sh dev
+
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev init --setup
+
+# Run the development server w/ autoreloading (but no bg workers)
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev manage runserver 0.0.0.0:8000 --debug --reload
+
+# Run the production server (with bg workers but no autoreloading)
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server
 
 # (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
 # When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@@ -1398,9 +1414,15 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 <details><summary><i>Click to expand...</i></summary>
 
 ```bash
+# set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
+
+# OR you can run a dev server with DEBUG=True in a few ways:
+archivebox manage runserver --debug --reload 0.0.0.0:8000
 # or
-archivebox server --debug ...
+archivebox server --debug 0.0.0.0:8000
+# or
+env DEBUG=True daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
 ```
 
 https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
@@ -1435,12 +1457,12 @@ services:
 
 # or with plain Docker:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
-docker run -it -v $PWD:/data archivebox:dev init --setup
+docker run -it -v $PWD:/data archivebox:dev init
 
 # or with pip:
 pip install 'git+https://github.com/pirate/ArchiveBox@dev'
 npm install 'git+https://github.com/ArchiveBox/ArchiveBox.git#dev'
-archivebox init --setup
+archivebox install
 ```
 
 </details>
@@ -1534,6 +1556,9 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 
 **The process to contribute a new extractor is like this:**
 
+> [!IMPORTANT]
+> This process is getting much easier after v0.8.x, there is a new plugin system under development: https://github.com/ArchiveBox/ArchiveBox/releases/tag/v0.8.4-rc
+
 1. [Open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new?assignees=&labels=changes%3A+behavior%2Cstatus%3A+idea+phase&template=feature_request.md&title=Feature+Request%3A+...) with your propsoed implementation (please link to the pages of any new external dependencies you plan on using)
 2. Ensure any dependencies needed are easily installable via a package managers like `apt`, `brew`, `pip3`, `npm`
    (Ideally, prefer to use external programs available via `pip3` or `npm`, however we do support using any binary installable via package manager that exposes a CLI/Python API and writes output to stdout or the filesystem.)
@@ -1562,8 +1587,6 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 # or individually:
 ./bin/build_docs.sh
 ./bin/build_pip.sh
-./bin/build_deb.sh
-./bin/build_brew.sh
 ./bin/build_docker.sh
 ```
 
@@ -1580,8 +1603,6 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 # or individually:
 ./bin/release_docs.sh
 ./bin/release_pip.sh
-./bin/release_deb.sh
-./bin/release_brew.sh
 ./bin/release_docker.sh
 ```
 
@@ -1608,14 +1629,14 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 <a href="https://github.com/sponsors/pirate"><img src="https://img.shields.io/badge/Github_Sponsors-%23B7CDFE.svg"/></a> &nbsp;
 <a href="https://www.patreon.com/theSquashSH"><img src="https://img.shields.io/badge/Patreon-%23DD5D76.svg"/></a> &nbsp;
 <a href="https://paypal.me/NicholasSweeting"><img src="https://img.shields.io/badge/Paypal-%23FFD141.svg"/></a> &nbsp;
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a> &nbsp;
+<a href="https://archivebox-shop.fourthwall.com/"><img src="https://img.shields.io/badge/Merch-%23903851.svg"/></a>
 <br/>
 <sup><i>ArchiveBox operates as a US 501(c)(3) nonprofit <a href="https://en.wikipedia.org/wiki/Fiscal_sponsorship">FSP</a> (sponsored by <a href="https://hackclub.com/hcb?ref=donation">HCB</a>), <a href="https://hcb.hackclub.com/donations/start/archivebox">direct donations</a> are tax-deductible.</i></sup>
 <br/><br/>
 <a href="https://twitter.com/ArchiveBoxApp"><img src="https://img.shields.io/badge/Tweet-%40ArchiveBoxApp-blue.svg?style=flat"/></a>&nbsp;
 <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?style=flat&label=Star+on+Github"/></a>&nbsp;
-<a href="https://zulip.archivebox.io/"><img src="https://img.shields.io/badge/Join_Our_Community-Zulip_Forum-%23B7EDFE.svg"/></a><br/>
-<sup>ArchiveBox was started by <a href="https://docs.sweeting.me/s/blog#About">Nick Sweeting</a> in 2017, and has <a href="https://star-history.com/#archivebox/archivebox&Date">grown steadily</a> with help from our <a href="https://github.com/ArchiveBox/ArchiveBox/graphs/contributors">amazing contributors</a>.</sup>
+<a href="https://zulip.archivebox.io/"><img src="https://img.shields.io/badge/Join_Our_Community-Zulip_Forum-%23B7EDFE.svg"/></a><br/><br/>
 <hr/>
 <i>✨ Have spare CPU/disk/bandwidth after all your 网站存档爬 and want to help the world?<br/>Check out our <a href="https://github.com/ArchiveBox/good-karma-kit">Good Karma Kit</a>...</i>
 </div>

TODO_archivebox_jsonl_cli.md

@@ -0,0 +1,716 @@
+# ArchiveBox CLI Pipeline Architecture
+
+## Overview
+
+This plan implements a JSONL-based CLI pipeline for ArchiveBox, enabling Unix-style piping between commands:
+
+```bash
+archivebox crawl create URL | archivebox snapshot create | archivebox archiveresult create | archivebox run
+```
+
+## Design Principles
+
+1. **Maximize model method reuse**: Use `.to_json()`, `.from_json()`, `.to_jsonl()`, `.from_jsonl()` everywhere
+2. **Pass-through behavior**: All commands output input records + newly created records (accumulating pipeline)
+3. **Create-or-update**: Commands create records if they don't exist, update if ID matches existing
+4. **Auto-cascade**: `archivebox run` automatically creates Snapshots from Crawls and ArchiveResults from Snapshots
+5. **Generic filtering**: Implement filters as functions that take queryset → return queryset
+6. **Minimal code**: Extract duplicated `apply_filters()` to shared module
+
+---
+
+## Real-World Use Cases
+
+These examples demonstrate the JSONL piping architecture. Key points:
+- `archivebox run` auto-cascades (Crawl → Snapshots → ArchiveResults)
+- `archivebox run` **emits JSONL** of everything it creates, enabling chained processing
+- Use CLI args (`--status=`, `--plugin=`) for efficient DB filtering; use jq for transforms
+
+### 1. Basic Archive
+```bash
+# Simple URL archive (run auto-creates snapshots and archive results)
+archivebox crawl create https://example.com | archivebox run
+
+# Multiple URLs from a file
+archivebox crawl create < urls.txt | archivebox run
+
+# With depth crawling (follow links)
+archivebox crawl create --depth=2 https://docs.python.org | archivebox run
+```
+
+### 2. Retry Failed Extractions
+```bash
+# Retry all failed extractions
+archivebox archiveresult list --status=failed | archivebox run
+
+# Retry only failed PDFs from a specific domain
+archivebox archiveresult list --status=failed --plugin=pdf --url__icontains=nytimes.com \
+  | archivebox run
+```
+
+### 3. Import Bookmarks from Pinboard (jq transform)
+```bash
+# Fetch Pinboard API, transform fields to match ArchiveBox schema, archive
+curl -s "https://api.pinboard.in/v1/posts/all?format=json&auth_token=$TOKEN" \
+  | jq -c '.[] | {url: .href, tags_str: .tags, title: .description}' \
+  | archivebox crawl create \
+  | archivebox run
+```
+
+### 4. Retry Failed with Different Binary (jq transform + re-run)
+```bash
+# Get failed wget results, transform to use wget2 binary instead, re-queue as new attempts
+archivebox archiveresult list --status=failed --plugin=wget \
+  | jq -c '{snapshot_id, plugin, status: "queued", overrides: {WGET_BINARY: "wget2"}}' \
+  | archivebox archiveresult create \
+  | archivebox run
+
+# Chain processing: archive, then re-run any failures with increased timeout
+archivebox crawl create https://slow-site.com \
+  | archivebox run \
+  | jq -c 'select(.type == "ArchiveResult" and .status == "failed")
+           | del(.id) | .status = "queued" | .overrides.TIMEOUT = "120"' \
+  | archivebox archiveresult create \
+  | archivebox run
+```
+
+### 5. Selective Extraction
+```bash
+# Create only screenshot extractions for queued snapshots
+archivebox snapshot list --status=queued \
+  | archivebox archiveresult create --plugin=screenshot \
+  | archivebox run
+
+# Re-run singlefile on everything that was skipped
+archivebox archiveresult list --plugin=singlefile --status=skipped \
+  | archivebox archiveresult update --status=queued \
+  | archivebox run
+```
+
+### 6. Bulk Tag Management
+```bash
+# Tag all Twitter/X URLs (efficient DB filter, no jq needed)
+archivebox snapshot list --url__icontains=twitter.com \
+  | archivebox snapshot update --tag=twitter
+
+# Tag snapshots based on computed criteria (jq for logic DB can't do)
+archivebox snapshot list --status=sealed \
+  | jq -c 'select(.archiveresult_count > 5) | . + {tags_str: (.tags_str + ",well-archived")}' \
+  | archivebox snapshot update
+```
+
+### 7. RSS Feed Monitoring
+```bash
+# Archive all items from an RSS feed
+curl -s "https://hnrss.org/frontpage" \
+  | xq -r '.rss.channel.item[].link' \
+  | archivebox crawl create --tag=hackernews-$(date +%Y%m%d) \
+  | archivebox run
+```
+
+### 8. Recursive Link Following (run output → filter → re-run)
+```bash
+# Archive a page, then archive all PDFs it links to
+archivebox crawl create https://research-papers.org/index.html \
+  | archivebox run \
+  | jq -c 'select(.type == "Snapshot") | .discovered_urls[]?
+           | select(endswith(".pdf")) | {url: .}' \
+  | archivebox crawl create --tag=linked-pdfs \
+  | archivebox run
+
+# Depth crawl with custom handling: retry timeouts with longer timeout
+archivebox crawl create --depth=1 https://example.com \
+  | archivebox run \
+  | jq -c 'select(.type == "ArchiveResult" and .status == "failed" and .error contains "timeout")
+           | del(.id) | .overrides.TIMEOUT = "300"' \
+  | archivebox archiveresult create \
+  | archivebox run
+```
+
+### Composability Summary
+
+| Pattern | Example |
+|---------|---------|
+| **Filter → Process** | `list --status=failed --plugin=pdf \| run` |
+| **Transform → Archive** | `curl API \| jq '{url, tags_str}' \| crawl create \| run` |
+| **Retry w/ Changes** | `run \| jq 'select(.status=="failed") \| del(.id)' \| create \| run` |
+| **Selective Extract** | `snapshot list \| archiveresult create --plugin=screenshot` |
+| **Bulk Update** | `list --url__icontains=X \| update --tag=Y` |
+| **Chain Processing** | `crawl \| run \| jq transform \| create \| run` |
+
+The key insight: **`archivebox run` emits JSONL of everything it creates**, enabling:
+- Retry failed items with different settings (timeouts, binaries, etc.)
+- Recursive crawling (archive page → extract links → archive those)
+- Chained transforms (filter failures, modify config, re-queue)
+
+---
+
+## Code Reuse Findings
+
+### Existing Model Methods (USE THESE)
+- `Crawl.to_json()`, `Crawl.from_json()`, `Crawl.to_jsonl()`, `Crawl.from_jsonl()`
+- `Snapshot.to_json()`, `Snapshot.from_json()`, `Snapshot.to_jsonl()`, `Snapshot.from_jsonl()`
+- `Tag.to_json()`, `Tag.from_json()`, `Tag.to_jsonl()`, `Tag.from_jsonl()`
+
+### Missing Model Methods (MUST IMPLEMENT)
+- **`ArchiveResult.from_json()`** - Does not exist, must be added
+- **`ArchiveResult.from_jsonl()`** - Does not exist, must be added
+
+### Existing Utilities (USE THESE)
+- `archivebox/misc/jsonl.py`: `read_stdin()`, `read_args_or_stdin()`, `write_record()`, `parse_line()`
+- Type constants: `TYPE_CRAWL`, `TYPE_SNAPSHOT`, `TYPE_ARCHIVERESULT`, etc.
+
+### Duplicated Code (EXTRACT)
+- `apply_filters()` duplicated in 7 CLI files → extract to `archivebox/cli/cli_utils.py`
+
+### Supervisord Config (UPDATE)
+- `archivebox/workers/supervisord_util.py` line ~35: `"command": "archivebox manage orchestrator"` → `"command": "archivebox run"`
+
+### Field Name Standardization (FIX)
+- **Issue**: `Crawl.to_json()` outputs `tags_str`, but `Snapshot.to_json()` outputs `tags`
+- **Fix**: Standardize all models to use `tags_str` in JSONL output (matches model property names)
+
+---
+
+## Implementation Order
+
+### Phase 1: Model Prerequisites
+1. **Implement `ArchiveResult.from_json()`** in `archivebox/core/models.py`
+   - Pattern: Match `Snapshot.from_json()` and `Crawl.from_json()` style
+   - Handle: ID lookup (update existing) or create new
+   - Required fields: `snapshot_id`, `plugin`
+   - Optional fields: `status`, `hook_name`, etc.
+
+2. **Implement `ArchiveResult.from_jsonl()`** in `archivebox/core/models.py`
+   - Filter records by `type='ArchiveResult'`
+   - Call `from_json()` for each matching record
+
+3. **Fix `Snapshot.to_json()` field name**
+   - Change `'tags': self.tags_str()` → `'tags_str': self.tags_str()`
+   - Update any code that depends on `tags` key in Snapshot JSONL
+
+### Phase 2: Shared Utilities
+4. **Extract `apply_filters()` to `archivebox/cli/cli_utils.py`**
+   - Generic queryset filtering from CLI kwargs
+   - Support `--id__in=[csv]`, `--url__icontains=str`, etc.
+   - Remove duplicates from 7 CLI files
+
+### Phase 3: Pass-Through Behavior (NEW FEATURE)
+5. **Add pass-through to `archivebox crawl create`**
+   - Output non-Crawl input records unchanged
+   - Output created Crawl records
+
+6. **Add pass-through to `archivebox snapshot create`**
+   - Output non-Snapshot/non-Crawl input records unchanged
+   - Process Crawl records → create Snapshots
+   - Output both original Crawl and created Snapshots
+
+7. **Add pass-through to `archivebox archiveresult create`**
+   - Output non-Snapshot/non-ArchiveResult input records unchanged
+   - Process Snapshot records → create ArchiveResults
+   - Output both original Snapshots and created ArchiveResults
+
+8. **Add create-or-update to `archivebox run`**
+   - Records WITH id: lookup and queue existing
+   - Records WITHOUT id: create via `Model.from_json()`, then queue
+   - Pass-through output of all processed records
+
+### Phase 4: Test Infrastructure
+9. **Create `archivebox/tests/conftest.py`** with pytest-django
+   - Use `pytest-django` for proper test database handling
+   - Isolated DATA_DIR per test via `tmp_path` fixture
+   - `run_archivebox_cmd()` helper for subprocess testing
+
+### Phase 5: Unit Tests
+10. **Create `archivebox/tests/test_cli_crawl.py`** - crawl create/list/pass-through tests
+11. **Create `archivebox/tests/test_cli_snapshot.py`** - snapshot create/list/pass-through tests
+12. **Create `archivebox/tests/test_cli_archiveresult.py`** - archiveresult create/list/pass-through tests
+13. **Create `archivebox/tests/test_cli_run.py`** - run command create-or-update tests
+
+### Phase 6: Integration & Config
+14. **Extend `archivebox/cli/tests_piping.py`** - Add pass-through integration tests
+15. **Update supervisord config** - `orchestrator` → `run`
+
+---
+
+## Future Work (Deferred)
+
+### Commands to Defer
+- `archivebox tag create|list|update|delete` - Already works, defer improvements
+- `archivebox binary create|list|update|delete` - Lower priority
+- `archivebox process list` - Lower priority
+- `archivebox apikey create|list|update|delete` - Lower priority
+
+### `archivebox add` Relationship
+- **Current**: `archivebox add` is the primary user-facing command, stays as-is
+- **Future**: Refactor `add` to internally use `crawl create | snapshot create | run` pipeline
+- **Note**: This refactor is deferred; `add` continues to work independently for now
+
+---
+
+## Key Files
+
+| File | Action | Phase |
+|------|--------|-------|
+| `archivebox/core/models.py` | Add `ArchiveResult.from_json()`, `from_jsonl()` | 1 |
+| `archivebox/core/models.py` | Fix `Snapshot.to_json()` → `tags_str` | 1 |
+| `archivebox/cli/cli_utils.py` | NEW - shared `apply_filters()` | 2 |
+| `archivebox/cli/archivebox_crawl.py` | Add pass-through to create | 3 |
+| `archivebox/cli/archivebox_snapshot.py` | Add pass-through to create | 3 |
+| `archivebox/cli/archivebox_archiveresult.py` | Add pass-through to create | 3 |
+| `archivebox/cli/archivebox_run.py` | Add create-or-update, pass-through | 3 |
+| `archivebox/tests/conftest.py` | NEW - pytest fixtures | 4 |
+| `archivebox/tests/test_cli_crawl.py` | NEW - crawl unit tests | 5 |
+| `archivebox/tests/test_cli_snapshot.py` | NEW - snapshot unit tests | 5 |
+| `archivebox/tests/test_cli_archiveresult.py` | NEW - archiveresult unit tests | 5 |
+| `archivebox/tests/test_cli_run.py` | NEW - run unit tests | 5 |
+| `archivebox/cli/tests_piping.py` | Extend with pass-through tests | 6 |
+| `archivebox/workers/supervisord_util.py` | Update orchestrator→run | 6 |
+
+---
+
+## Implementation Details
+
+### ArchiveResult.from_json() Design
+
+```python
+@staticmethod
+def from_json(record: Dict[str, Any], overrides: Dict[str, Any] = None) -> 'ArchiveResult | None':
+    """
+    Create or update a single ArchiveResult from a JSON record dict.
+
+    Args:
+        record: Dict with 'snapshot_id' and 'plugin' (required for create),
+                or 'id' (for update)
+        overrides: Dict of field overrides
+
+    Returns:
+        ArchiveResult instance or None if invalid
+    """
+    from django.utils import timezone
+
+    overrides = overrides or {}
+
+    # If 'id' is provided, lookup and update existing
+    result_id = record.get('id')
+    if result_id:
+        try:
+            result = ArchiveResult.objects.get(id=result_id)
+            # Update fields from record
+            if record.get('status'):
+                result.status = record['status']
+                result.retry_at = timezone.now()
+            result.save()
+            return result
+        except ArchiveResult.DoesNotExist:
+            pass  # Fall through to create
+
+    # Required fields for creation
+    snapshot_id = record.get('snapshot_id')
+    plugin = record.get('plugin')
+
+    if not snapshot_id or not plugin:
+        return None
+
+    try:
+        snapshot = Snapshot.objects.get(id=snapshot_id)
+    except Snapshot.DoesNotExist:
+        return None
+
+    # Create or get existing result
+    result, created = ArchiveResult.objects.get_or_create(
+        snapshot=snapshot,
+        plugin=plugin,
+        defaults={
+            'status': record.get('status', ArchiveResult.StatusChoices.QUEUED),
+            'retry_at': timezone.now(),
+            'hook_name': record.get('hook_name', ''),
+            **overrides,
+        }
+    )
+
+    # If not created, optionally reset for retry
+    if not created and record.get('status'):
+        result.status = record['status']
+        result.retry_at = timezone.now()
+        result.save()
+
+    return result
+```
+
+### Pass-Through Pattern
+
+All `create` commands follow this pattern:
+
+```python
+def create_X(args, ...):
+    is_tty = sys.stdout.isatty()
+    records = list(read_args_or_stdin(args))
+
+    for record in records:
+        record_type = record.get('type')
+
+        # Pass-through: output records we don't handle
+        if record_type not in HANDLED_TYPES:
+            if not is_tty:
+                write_record(record)
+            continue
+
+        # Handle our type: create via Model.from_json()
+        obj = Model.from_json(record, overrides={...})
+
+        # Output created record (hydrated with db id)
+        if obj and not is_tty:
+            write_record(obj.to_json())
+```
+
+### Pass-Through Semantics Example
+
+```
+Input:
+  {"type": "Crawl", "id": "abc", "urls": "https://example.com", ...}
+  {"type": "Tag", "name": "important"}
+
+archivebox snapshot create output:
+  {"type": "Crawl", "id": "abc", ...}           # pass-through (not our type)
+  {"type": "Tag", "name": "important"}          # pass-through (not our type)
+  {"type": "Snapshot", "id": "xyz", ...}        # created from Crawl URLs
+```
+
+### Create-or-Update Pattern for `archivebox run`
+
+```python
+def process_stdin_records() -> int:
+    records = list(read_stdin())
+    is_tty = sys.stdout.isatty()
+
+    for record in records:
+        record_type = record.get('type')
+        record_id = record.get('id')
+
+        # Create-or-update based on whether ID exists
+        if record_type == TYPE_CRAWL:
+            if record_id:
+                try:
+                    obj = Crawl.objects.get(id=record_id)
+                except Crawl.DoesNotExist:
+                    obj = Crawl.from_json(record)
+            else:
+                obj = Crawl.from_json(record)
+
+            if obj:
+                obj.retry_at = timezone.now()
+                obj.save()
+                if not is_tty:
+                    write_record(obj.to_json())
+
+        # Similar for Snapshot, ArchiveResult...
+```
+
+### Shared apply_filters() Design
+
+Extract to `archivebox/cli/cli_utils.py`:
+
+```python
+"""Shared CLI utilities for ArchiveBox commands."""
+
+from typing import Optional
+
+def apply_filters(queryset, filter_kwargs: dict, limit: Optional[int] = None):
+    """
+    Apply Django-style filters from CLI kwargs to a QuerySet.
+
+    Supports: --status=queued, --url__icontains=example, --id__in=uuid1,uuid2
+
+    Args:
+        queryset: Django QuerySet to filter
+        filter_kwargs: Dict of filter key-value pairs from CLI
+        limit: Optional limit on results
+
+    Returns:
+        Filtered QuerySet
+    """
+    filters = {}
+    for key, value in filter_kwargs.items():
+        if value is None or key in ('limit', 'offset'):
+            continue
+        # Handle CSV lists for __in filters
+        if key.endswith('__in') and isinstance(value, str):
+            value = [v.strip() for v in value.split(',')]
+        filters[key] = value
+
+    if filters:
+        queryset = queryset.filter(**filters)
+    if limit:
+        queryset = queryset[:limit]
+
+    return queryset
+```
+
+---
+
+## conftest.py Design (pytest-django)
+
+```python
+"""archivebox/tests/conftest.py - Pytest fixtures for CLI tests."""
+
+import os
+import sys
+import json
+import subprocess
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Tuple
+
+import pytest
+
+
+# =============================================================================
+# Fixtures
+# =============================================================================
+
+@pytest.fixture
+def isolated_data_dir(tmp_path, settings):
+    """
+    Create isolated DATA_DIR for each test.
+
+    Uses tmp_path for isolation, configures Django settings.
+    """
+    data_dir = tmp_path / 'archivebox_data'
+    data_dir.mkdir()
+
+    # Set environment for subprocess calls
+    os.environ['DATA_DIR'] = str(data_dir)
+
+    # Update Django settings
+    settings.DATA_DIR = data_dir
+
+    yield data_dir
+
+    # Cleanup handled by tmp_path fixture
+
+
+@pytest.fixture
+def initialized_archive(isolated_data_dir):
+    """
+    Initialize ArchiveBox archive in isolated directory.
+
+    Runs `archivebox init` to set up database and directories.
+    """
+    from archivebox.cli.archivebox_init import init
+    init(setup=True, quick=True)
+    return isolated_data_dir
+
+
+@pytest.fixture
+def cli_env(initialized_archive):
+    """
+    Environment dict for CLI subprocess calls.
+
+    Includes DATA_DIR and disables slow extractors.
+    """
+    return {
+        **os.environ,
+        'DATA_DIR': str(initialized_archive),
+        'USE_COLOR': 'False',
+        'SHOW_PROGRESS': 'False',
+        'SAVE_TITLE': 'True',
+        'SAVE_FAVICON': 'False',
+        'SAVE_WGET': 'False',
+        'SAVE_WARC': 'False',
+        'SAVE_PDF': 'False',
+        'SAVE_SCREENSHOT': 'False',
+        'SAVE_DOM': 'False',
+        'SAVE_SINGLEFILE': 'False',
+        'SAVE_READABILITY': 'False',
+        'SAVE_MERCURY': 'False',
+        'SAVE_GIT': 'False',
+        'SAVE_YTDLP': 'False',
+        'SAVE_HEADERS': 'False',
+    }
+
+
+# =============================================================================
+# CLI Helpers
+# =============================================================================
+
+def run_archivebox_cmd(
+    args: List[str],
+    stdin: Optional[str] = None,
+    cwd: Optional[Path] = None,
+    env: Optional[Dict[str, str]] = None,
+    timeout: int = 60,
+) -> Tuple[str, str, int]:
+    """
+    Run archivebox command, return (stdout, stderr, returncode).
+
+    Args:
+        args: Command arguments (e.g., ['crawl', 'create', 'https://example.com'])
+        stdin: Optional string to pipe to stdin
+        cwd: Working directory (defaults to DATA_DIR from env)
+        env: Environment variables (defaults to os.environ with DATA_DIR)
+        timeout: Command timeout in seconds
+
+    Returns:
+        Tuple of (stdout, stderr, returncode)
+    """
+    cmd = [sys.executable, '-m', 'archivebox'] + args
+
+    env = env or {**os.environ}
+    cwd = cwd or Path(env.get('DATA_DIR', '.'))
+
+    result = subprocess.run(
+        cmd,
+        input=stdin,
+        capture_output=True,
+        text=True,
+        cwd=cwd,
+        env=env,
+        timeout=timeout,
+    )
+
+    return result.stdout, result.stderr, result.returncode
+
+
+# =============================================================================
+# Output Assertions
+# =============================================================================
+
+def parse_jsonl_output(stdout: str) -> List[Dict[str, Any]]:
+    """Parse JSONL output into list of dicts."""
+    records = []
+    for line in stdout.strip().split('\n'):
+        line = line.strip()
+        if line and line.startswith('{'):
+            try:
+                records.append(json.loads(line))
+            except json.JSONDecodeError:
+                pass
+    return records
+
+
+def assert_jsonl_contains_type(stdout: str, record_type: str, min_count: int = 1):
+    """Assert output contains at least min_count records of type."""
+    records = parse_jsonl_output(stdout)
+    matching = [r for r in records if r.get('type') == record_type]
+    assert len(matching) >= min_count, \
+        f"Expected >= {min_count} {record_type}, got {len(matching)}"
+    return matching
+
+
+def assert_jsonl_pass_through(stdout: str, input_records: List[Dict[str, Any]]):
+    """Assert that input records appear in output (pass-through behavior)."""
+    output_records = parse_jsonl_output(stdout)
+    output_ids = {r.get('id') for r in output_records if r.get('id')}
+
+    for input_rec in input_records:
+        input_id = input_rec.get('id')
+        if input_id:
+            assert input_id in output_ids, \
+                f"Input record {input_id} not found in output (pass-through failed)"
+
+
+def assert_record_has_fields(record: Dict[str, Any], required_fields: List[str]):
+    """Assert record has all required fields with non-None values."""
+    for field in required_fields:
+        assert field in record, f"Record missing field: {field}"
+        assert record[field] is not None, f"Record field is None: {field}"
+
+
+# =============================================================================
+# Database Assertions
+# =============================================================================
+
+def assert_db_count(model_class, filters: Dict[str, Any], expected: int):
+    """Assert database count matches expected."""
+    actual = model_class.objects.filter(**filters).count()
+    assert actual == expected, \
+        f"Expected {expected} {model_class.__name__}, got {actual}"
+
+
+def assert_db_exists(model_class, **filters):
+    """Assert at least one record exists matching filters."""
+    assert model_class.objects.filter(**filters).exists(), \
+        f"No {model_class.__name__} found matching {filters}"
+
+
+# =============================================================================
+# Test Data Factories
+# =============================================================================
+
+def create_test_url(domain: str = 'example.com', path: str = None) -> str:
+    """Generate unique test URL."""
+    import uuid
+    path = path or uuid.uuid4().hex[:8]
+    return f'https://{domain}/{path}'
+
+
+def create_test_crawl_json(urls: List[str] = None, **kwargs) -> Dict[str, Any]:
+    """Create Crawl JSONL record for testing."""
+    from archivebox.misc.jsonl import TYPE_CRAWL
+
+    urls = urls or [create_test_url()]
+    return {
+        'type': TYPE_CRAWL,
+        'urls': '\n'.join(urls),
+        'max_depth': kwargs.get('max_depth', 0),
+        'tags_str': kwargs.get('tags_str', ''),
+        'status': kwargs.get('status', 'queued'),
+        **{k: v for k, v in kwargs.items() if k not in ('max_depth', 'tags_str', 'status')},
+    }
+
+
+def create_test_snapshot_json(url: str = None, **kwargs) -> Dict[str, Any]:
+    """Create Snapshot JSONL record for testing."""
+    from archivebox.misc.jsonl import TYPE_SNAPSHOT
+
+    return {
+        'type': TYPE_SNAPSHOT,
+        'url': url or create_test_url(),
+        'tags_str': kwargs.get('tags_str', ''),
+        'status': kwargs.get('status', 'queued'),
+        **{k: v for k, v in kwargs.items() if k not in ('tags_str', 'status')},
+    }
+```
+
+---
+
+## Test Rules
+
+- **NO SKIPPING** - Every test runs
+- **NO MOCKING** - Real subprocess calls, real database
+- **NO DISABLING** - Failing tests identify real problems
+- **MINIMAL CODE** - Import helpers from conftest.py
+- **ISOLATED** - Each test gets its own DATA_DIR via `tmp_path`
+
+---
+
+## Task Checklist
+
+### Phase 1: Model Prerequisites
+- [x] Implement `ArchiveResult.from_json()` in `archivebox/core/models.py`
+- [x] Implement `ArchiveResult.from_jsonl()` in `archivebox/core/models.py`
+- [x] Fix `Snapshot.to_json()` to use `tags_str` instead of `tags`
+
+### Phase 2: Shared Utilities
+- [x] Create `archivebox/cli/cli_utils.py` with shared `apply_filters()`
+- [x] Update 7 CLI files to import from `cli_utils.py`
+
+### Phase 3: Pass-Through Behavior
+- [x] Add pass-through to `archivebox_crawl.py` create
+- [x] Add pass-through to `archivebox_snapshot.py` create
+- [x] Add pass-through to `archivebox_archiveresult.py` create
+- [x] Add create-or-update to `archivebox_run.py`
+- [x] Add pass-through output to `archivebox_run.py`
+
+### Phase 4: Test Infrastructure
+- [x] Create `archivebox/tests/conftest.py` with pytest-django fixtures
+
+### Phase 5: Unit Tests
+- [x] Create `archivebox/tests/test_cli_crawl.py`
+- [x] Create `archivebox/tests/test_cli_snapshot.py`
+- [x] Create `archivebox/tests/test_cli_archiveresult.py`
+- [x] Create `archivebox/tests/test_cli_run.py`
+
+### Phase 6: Integration & Config
+- [x] Extend `archivebox/cli/tests_piping.py` with pass-through tests
+- [x] Update `archivebox/workers/supervisord_util.py`: orchestrator→run

TODO_cli_refactor.md

@@ -0,0 +1,131 @@
+# ArchiveBox CLI Refactor TODO
+
+## Design Decisions
+
+1. **Keep `archivebox add`** as high-level convenience command
+2. **Unified `archivebox run`** for processing (replaces per-model `run` and `orchestrator`)
+3. **Expose all models** including binary, process, machine
+4. **Clean break** from old command structure (no backward compatibility aliases)
+
+## Final Architecture
+
+```
+archivebox <model> <action> [args...] [--filters]
+archivebox run [stdin JSONL]
+```
+
+### Actions (4 per model):
+- `create` - Create records (from args, stdin, or JSONL), dedupes by indexed fields
+- `list` - Query records (with filters, returns JSONL)
+- `update` - Modify records (from stdin JSONL, PATCH semantics)
+- `delete` - Remove records (from stdin JSONL, requires --yes)
+
+### Unified Run Command:
+- `archivebox run` - Process queued work
+  - With stdin JSONL: Process piped records, exit when complete
+  - Without stdin (TTY): Run orchestrator in foreground until killed
+
+### Models (7 total):
+- `crawl` - Crawl jobs
+- `snapshot` - Individual archived pages
+- `archiveresult` - Plugin extraction results
+- `tag` - Tags/labels
+- `binary` - Detected binaries (chrome, wget, etc.)
+- `process` - Process execution records (read-only)
+- `machine` - Machine/host records (read-only)
+
+---
+
+## Implementation Checklist
+
+### Phase 1: Unified Run Command
+- [x] Create `archivebox/cli/archivebox_run.py` - unified processing command
+
+### Phase 2: Core Model Commands
+- [x] Refactor `archivebox/cli/archivebox_snapshot.py` to Click group with create|list|update|delete
+- [x] Refactor `archivebox/cli/archivebox_crawl.py` to Click group with create|list|update|delete
+- [x] Create `archivebox/cli/archivebox_archiveresult.py` with create|list|update|delete
+- [x] Create `archivebox/cli/archivebox_tag.py` with create|list|update|delete
+
+### Phase 3: System Model Commands
+- [x] Create `archivebox/cli/archivebox_binary.py` with create|list|update|delete
+- [x] Create `archivebox/cli/archivebox_process.py` with list only (read-only)
+- [x] Create `archivebox/cli/archivebox_machine.py` with list only (read-only)
+
+### Phase 4: Registry & Cleanup
+- [x] Update `archivebox/cli/__init__.py` command registry
+- [x] Delete `archivebox/cli/archivebox_extract.py`
+- [x] Delete `archivebox/cli/archivebox_remove.py`
+- [x] Delete `archivebox/cli/archivebox_search.py`
+- [x] Delete `archivebox/cli/archivebox_orchestrator.py`
+- [x] Update `archivebox/cli/archivebox_add.py` internals (no changes needed - uses models directly)
+- [x] Update `archivebox/cli/tests_piping.py`
+
+### Phase 5: Tests for New Commands
+- [ ] Add tests for `archivebox run` command
+- [ ] Add tests for `archivebox crawl create|list|update|delete`
+- [ ] Add tests for `archivebox snapshot create|list|update|delete`
+- [ ] Add tests for `archivebox archiveresult create|list|update|delete`
+- [ ] Add tests for `archivebox tag create|list|update|delete`
+- [ ] Add tests for `archivebox binary create|list|update|delete`
+- [ ] Add tests for `archivebox process list`
+- [ ] Add tests for `archivebox machine list`
+
+---
+
+## Usage Examples
+
+### Basic CRUD
+```bash
+# Create
+archivebox crawl create https://example.com https://foo.com --depth=1
+archivebox snapshot create https://example.com --tag=news
+
+# List with filters
+archivebox crawl list --status=queued
+archivebox snapshot list --url__icontains=example.com
+archivebox archiveresult list --status=failed --plugin=screenshot
+
+# Update (reads JSONL from stdin, applies changes)
+archivebox snapshot list --tag=old | archivebox snapshot update --tag=new
+
+# Delete (requires --yes)
+archivebox crawl list --url__icontains=example.com | archivebox crawl delete --yes
+```
+
+### Unified Run Command
+```bash
+# Run orchestrator in foreground (replaces `archivebox orchestrator`)
+archivebox run
+
+# Process specific records (pipe any JSONL type, exits when done)
+archivebox snapshot list --status=queued | archivebox run
+archivebox archiveresult list --status=failed | archivebox run
+archivebox crawl list --status=queued | archivebox run
+
+# Mixed types work too - run handles any JSONL
+cat mixed_records.jsonl | archivebox run
+```
+
+### Composed Workflows
+```bash
+# Full pipeline (replaces old `archivebox add`)
+archivebox crawl create https://example.com --status=queued \
+  | archivebox snapshot create --status=queued \
+  | archivebox archiveresult create --status=queued \
+  | archivebox run
+
+# Re-run failed extractions
+archivebox archiveresult list --status=failed | archivebox run
+
+# Delete all snapshots for a domain
+archivebox snapshot list --url__icontains=spam.com | archivebox snapshot delete --yes
+```
+
+### Keep `archivebox add` as convenience
+```bash
+# This remains the simple user-friendly interface:
+archivebox add https://example.com --depth=1 --tag=news
+
+# Internally equivalent to the composed pipeline above
+```

TODO_hook_concurrency.md

@@ -0,0 +1,532 @@
+# ArchiveBox Hook Script Concurrency & Execution Plan
+
+## Overview
+
+Snapshot.run() should enforce that snapshot hooks are run in **10 discrete, sequential "steps"**: `0*`, `1*`, `2*`, `3*`, `4*`, `5*`, `6*`, `7*`, `8*`, `9*`.
+
+For every discovered hook script, ArchiveBox should create an ArchiveResult in `queued` state, then manage running them using `retry_at` and inline logic to enforce this ordering.
+
+## Design Decisions
+
+### ArchiveResult Schema
+- Add `ArchiveResult.hook_name` (CharField, nullable) - just filename, e.g., `'on_Snapshot__20_chrome_tab.bg.js'`
+- Keep `ArchiveResult.plugin` - still important (plugin directory name)
+- Step number derived on-the-fly from `hook_name` via `extract_step(hook_name)` - not stored
+
+### Snapshot Schema
+- Add `Snapshot.current_step` (IntegerField 0-9, default=0)
+- Integrate with `SnapshotMachine` state transitions for step advancement
+
+### Hook Discovery & Execution
+- `Snapshot.run()` discovers all hooks upfront, creates one AR per hook with `hook_name` set
+- All ARs for a given step can be claimed and executed in parallel by workers
+- Workers claim ARs where `extract_step(ar.hook_name) <= snapshot.current_step`
+- `Snapshot.advance_step_if_ready()` increments `current_step` when:
+  - All **foreground** hooks in current step are finished (SUCCEEDED/FAILED/SKIPPED)
+  - Background hooks don't block advancement (they continue running)
+  - Called from `SnapshotMachine` state transitions
+
+### ArchiveResult.run() Behavior
+- If `self.hook_name` is set: run that single hook
+- If `self.hook_name` is None: discover all hooks for `self.plugin` and run sequentially
+- Background hooks detected by `.bg.` in filename (e.g., `on_Snapshot__20_chrome_tab.bg.js`)
+- Background hooks return immediately (ArchiveResult stays in STARTED state)
+- Foreground hooks wait for completion, update status from JSONL output
+
+### Hook Execution Flow
+1. **Within a step**: Workers claim all ARs for current step in parallel
+2. **Foreground hooks** (no .bg): ArchiveResult waits for completion, transitions to SUCCEEDED/FAILED/SKIPPED
+3. **Background hooks** (.bg): ArchiveResult transitions to STARTED, hook continues running
+4. **Step advancement**: `Snapshot.advance_step_if_ready()` checks:
+   - Are all foreground ARs in current step finished? (SUCCEEDED/FAILED/SKIPPED)
+   - Ignore ARs still in STARTED (background hooks)
+   - If yes, increment `current_step`
+5. **Snapshot sealing**: When `current_step=9` and all foreground hooks done, kill background hooks via `Snapshot.cleanup()`
+
+### Unnumbered Hooks
+- Extract step via `re.search(r'__(\d{2})_', hook_name)`, default to 9 if no match
+- Log warning for unnumbered hooks
+- Purely runtime derivation - no stored field
+
+## Hook Numbering Convention
+
+Hooks scripts are numbered `00` to `99` to control:
+- **First digit (0-9)**: Which step they are part of
+- **Second digit (0-9)**: Order within that step
+
+Hook scripts are launched **strictly sequentially** based on their filename alphabetical order, and run in sets of several per step before moving on to the next step.
+
+**Naming Format:**
+```
+on_{ModelName}__{run_order}_{human_readable_description}[.bg].{ext}
+```
+
+**Examples:**
+```
+on_Snapshot__00_this_would_run_first.sh
+on_Snapshot__05_start_ytdlp_download.bg.sh
+on_Snapshot__10_chrome_tab_opened.js
+on_Snapshot__50_screenshot.js
+on_Snapshot__53_media.bg.py
+```
+
+## Background (.bg) vs Foreground Scripts
+
+### Foreground Scripts (no .bg suffix)
+- Launch in parallel with other hooks in their step
+- Step waits for all foreground hooks to complete or timeout
+- Get killed with SIGTERM if they exceed their `PLUGINNAME_TIMEOUT`
+- Step advances when all foreground hooks finish
+
+### Background Scripts (.bg suffix)
+- Launch in parallel with other hooks in their step
+- Do NOT block step progression - step can advance while they run
+- Continue running across step boundaries until complete or timeout
+- Get killed with SIGTERM when Snapshot transitions to SEALED (via `Snapshot.cleanup()`)
+- Should exit naturally when work is complete (best case)
+
+**Important:** A .bg script started in step 2 can keep running through steps 3, 4, 5... until the Snapshot seals or the hook exits naturally.
+
+## Execution Step Guidelines
+
+These are **naming conventions and guidelines**, not enforced checkpoints. They provide semantic organization for plugin ordering:
+
+### Step 0: Pre-Setup
+```
+00-09: Initial setup, validation, feature detection
+```
+
+### Step 1: Chrome Launch & Tab Creation
+```
+10-19: Browser/tab lifecycle setup
+- Chrome browser launch
+- Tab creation and CDP connection
+```
+
+### Step 2: Navigation & Settlement
+```
+20-29: Page loading and settling
+- Navigate to URL
+- Wait for page load
+- Initial response capture (responses, ssl, consolelog as .bg listeners)
+```
+
+### Step 3: Page Adjustment
+```
+30-39: DOM manipulation before archiving
+- Hide popups/banners
+- Solve captchas
+- Expand comments/details sections
+- Inject custom CSS/JS
+- Accessibility modifications
+```
+
+### Step 4: Ready for Archiving
+```
+40-49: Final pre-archiving checks
+- Verify page is fully adjusted
+- Wait for any pending modifications
+```
+
+### Step 5: DOM Extraction (Sequential, Non-BG)
+```
+50-59: Extractors that need exclusive DOM access
+- singlefile (MUST NOT be .bg)
+- screenshot (MUST NOT be .bg)
+- pdf (MUST NOT be .bg)
+- dom (MUST NOT be .bg)
+- title
+- headers
+- readability
+- mercury
+
+These MUST run sequentially as they temporarily modify the DOM
+during extraction, then revert it. Running in parallel would corrupt results.
+```
+
+### Step 6: Post-DOM Extraction
+```
+60-69: Extractors that don't need DOM or run on downloaded files
+- wget
+- git
+- media (.bg - can run for hours)
+- gallerydl (.bg)
+- forumdl (.bg)
+- papersdl (.bg)
+```
+
+### Step 7: Chrome Cleanup
+```
+70-79: Browser/tab teardown
+- Close tabs
+- Cleanup Chrome resources
+```
+
+### Step 8: Post-Processing
+```
+80-89: Reprocess outputs from earlier extractors
+- OCR of images
+- Audio/video transcription
+- URL parsing from downloaded content (rss, html, json, txt, csv, md)
+- LLM analysis/summarization of outputs
+```
+
+### Step 9: Indexing & Finalization
+```
+90-99: Save to indexes and finalize
+- Index text content to Sonic/SQLite FTS
+- Create symlinks
+- Generate merkle trees
+- Final status updates
+```
+
+## Hook Script Interface
+
+### Input: CLI Arguments (NOT stdin)
+Hooks receive configuration as CLI flags (CSV or JSON-encoded):
+
+```bash
+--url="https://example.com"
+--snapshot-id="1234-5678-uuid"
+--config='{"some_key": "some_value"}'
+--plugins=git,media,favicon,title
+--timeout=50
+--enable-something
+```
+
+### Input: Environment Variables
+All configuration comes from env vars, defined in `plugin_dir/config.json` JSONSchema:
+
+```bash
+WGET_BINARY=/usr/bin/wget
+WGET_TIMEOUT=60
+WGET_USER_AGENT="Mozilla/5.0..."
+WGET_EXTRA_ARGS="--no-check-certificate"
+SAVE_WGET=True
+```
+
+**Required:** Every plugin must support `PLUGINNAME_TIMEOUT` for self-termination.
+
+### Output: Filesystem (CWD)
+Hooks read/write files to:
+- `$CWD`: Their own output subdirectory (e.g., `archive/snapshots/{id}/wget/`)
+- `$CWD/..`: Parent directory (to read outputs from other hooks)
+
+This allows hooks to:
+- Access files created by other hooks
+- Keep their outputs separate by default
+- Use semaphore files for coordination (if needed)
+
+### Output: JSONL to stdout
+Hooks emit one JSONL line per database record they want to create or update:
+
+```jsonl
+{"type": "Tag", "name": "sci-fi"}
+{"type": "ArchiveResult", "id": "1234-uuid", "status": "succeeded", "output_str": "wget/index.html"}
+{"type": "Snapshot", "id": "5678-uuid", "title": "Example Page"}
+```
+
+See `archivebox/misc/jsonl.py` and model `from_json()` / `from_jsonl()` methods for full list of supported types and fields.
+
+### Output: stderr for Human Logs
+Hooks should emit human-readable output or debug info to **stderr**. There are no guarantees this will be persisted long-term. Use stdout JSONL or filesystem for outputs that matter.
+
+### Cleanup: Delete Cruft
+If hooks emit no meaningful long-term outputs, they should delete any temporary files themselves to avoid wasting space. However, the ArchiveResult DB row should be kept so we know:
+- It doesn't need to be retried
+- It isn't missing
+- What happened (status, error message)
+
+### Signal Handling: SIGINT/SIGTERM
+Hooks are expected to listen for polite `SIGINT`/`SIGTERM` and finish hastily, then exit cleanly. Beyond that, they may be `SIGKILL'd` at ArchiveBox's discretion.
+
+**If hooks double-fork or spawn long-running processes:** They must output a `.pid` file in their directory so zombies can be swept safely.
+
+## Hook Failure Modes & Retry Logic
+
+Hooks can fail in several ways. ArchiveBox handles each differently:
+
+### 1. Soft Failure (Record & Don't Retry)
+**Exit:** `0` (success)
+**JSONL:** `{"type": "ArchiveResult", "status": "failed", "output_str": "404 Not Found"}`
+
+This means: "I ran successfully, but the resource wasn't available." Don't retry this.
+
+**Use cases:**
+- 404 errors
+- Content not available
+- Feature not applicable to this URL
+
+### 2. Hard Failure / Temporary Error (Retry Later)
+**Exit:** Non-zero (1, 2, etc.)
+**JSONL:** None (or incomplete)
+
+This means: "Something went wrong, I couldn't complete." Treat this ArchiveResult as "missing" and set `retry_at` for later.
+
+**Use cases:**
+- 500 server errors
+- Network timeouts
+- Binary not found / crashed
+- Transient errors
+
+**Behavior:**
+- ArchiveBox sets `retry_at` on the ArchiveResult
+- Hook will be retried during next `archivebox update`
+
+### 3. Partial Success (Update & Continue)
+**Exit:** Non-zero
+**JSONL:** Partial records emitted before crash
+
+**Behavior:**
+- Update ArchiveResult with whatever was emitted
+- Mark remaining work as "missing" with `retry_at`
+
+### 4. Success (Record & Continue)
+**Exit:** `0`
+**JSONL:** `{"type": "ArchiveResult", "status": "succeeded", "output_str": "output/file.html"}`
+
+This is the happy path.
+
+### Error Handling Rules
+
+- **DO NOT skip hooks** based on failures
+- **Continue to next hook** regardless of foreground or background failures
+- **Update ArchiveResults** with whatever information is available
+- **Set retry_at** for "missing" or temporarily-failed hooks
+- **Let background scripts continue** even if foreground scripts fail
+
+## File Structure
+
+```
+archivebox/plugins/{plugin_name}/
+├── config.json              # JSONSchema: env var config options
+├── binaries.jsonl           # Runtime dependencies: apt|brew|pip|npm|env
+├── on_Snapshot__XX_name.py  # Hook script (foreground)
+├── on_Snapshot__XX_name.bg.py  # Hook script (background)
+└── tests/
+    └── test_name.py
+```
+
+## Implementation Checklist
+
+### Phase 1: Schema Migration ✅
+- [x] Add `Snapshot.current_step` (IntegerField 0-9, default=0)
+- [x] Add `ArchiveResult.hook_name` (CharField, nullable) - just filename
+- [x] Create migration: `0034_snapshot_current_step.py`
+
+### Phase 2: Core Logic Updates ✅
+- [x] Add `extract_step(hook_name)` utility in `archivebox/hooks.py`
+  - Extract first digit from `__XX_` pattern
+  - Default to 9 for unnumbered hooks
+- [x] Add `is_background_hook(hook_name)` utility in `archivebox/hooks.py`
+  - Check for `.bg.` in filename
+- [x] Update `Snapshot.create_pending_archiveresults()` in `archivebox/core/models.py`:
+  - Discover all hooks (not plugins)
+  - Create one AR per hook with `hook_name` set
+- [x] Update `ArchiveResult.run()` in `archivebox/core/models.py`:
+  - If `hook_name` set: run single hook
+  - If `hook_name` None: discover all plugin hooks (existing behavior)
+- [x] Add `Snapshot.advance_step_if_ready()` method:
+  - Check if all foreground ARs in current step finished
+  - Increment `current_step` if ready
+  - Ignore background hooks (.bg) in completion check
+- [x] Integrate with `SnapshotMachine.is_finished()` in `archivebox/core/statemachines.py`:
+  - Call `advance_step_if_ready()` before checking if done
+
+### Phase 3: Worker Coordination ✅
+- [x] Update worker AR claiming query in `archivebox/workers/worker.py`:
+  - Filter: `extract_step(ar.hook_name) <= snapshot.current_step`
+  - Claims ARs in QUEUED state, checks step in Python before processing
+  - Orders by hook_name for deterministic execution within step
+
+### Phase 4: Hook Renumbering ✅
+- [x] Renumber hooks per renumbering map below
+- [x] Add `.bg` suffix to long-running hooks (media, gallerydl, forumdl, papersdl)
+- [x] Move parse_* hooks to step 7 (70-79)
+- [x] Test all hooks still work after renumbering
+
+## Migration Path
+
+### Natural Compatibility
+No special migration needed:
+1. Existing ARs with `hook_name=None` continue to work (discover all plugin hooks at runtime)
+2. New ARs get `hook_name` set (single hook per AR)
+3. `ArchiveResult.run()` handles both cases naturally
+4. Unnumbered hooks default to step 9 (log warning)
+
+### Renumbering Map
+
+**Completed Renames:**
+```
+# Step 5: DOM Extraction (sequential, non-background)
+singlefile/on_Snapshot__37_singlefile.py      → singlefile/on_Snapshot__50_singlefile.py ✅
+screenshot/on_Snapshot__34_screenshot.js      → screenshot/on_Snapshot__51_screenshot.js ✅
+pdf/on_Snapshot__35_pdf.js                    → pdf/on_Snapshot__52_pdf.js ✅
+dom/on_Snapshot__36_dom.js                    → dom/on_Snapshot__53_dom.js ✅
+title/on_Snapshot__32_title.js                → title/on_Snapshot__54_title.js ✅
+readability/on_Snapshot__52_readability.py    → readability/on_Snapshot__55_readability.py ✅
+headers/on_Snapshot__33_headers.js            → headers/on_Snapshot__55_headers.js ✅
+mercury/on_Snapshot__53_mercury.py            → mercury/on_Snapshot__56_mercury.py ✅
+htmltotext/on_Snapshot__54_htmltotext.py      → htmltotext/on_Snapshot__57_htmltotext.py ✅
+
+# Step 6: Post-DOM Extraction (background for long-running)
+wget/on_Snapshot__50_wget.py                  → wget/on_Snapshot__61_wget.py ✅
+git/on_Snapshot__12_git.py                    → git/on_Snapshot__62_git.py ✅
+media/on_Snapshot__51_media.py                → media/on_Snapshot__63_media.bg.py ✅
+gallerydl/on_Snapshot__52_gallerydl.py        → gallerydl/on_Snapshot__64_gallerydl.bg.py ✅
+forumdl/on_Snapshot__53_forumdl.py            → forumdl/on_Snapshot__65_forumdl.bg.py ✅
+papersdl/on_Snapshot__54_papersdl.py          → papersdl/on_Snapshot__66_papersdl.bg.py ✅
+
+# Step 7: URL Extraction (parse_* hooks moved from step 6)
+parse_html_urls/on_Snapshot__60_parse_html_urls.py      → parse_html_urls/on_Snapshot__70_parse_html_urls.py ✅
+parse_txt_urls/on_Snapshot__62_parse_txt_urls.py        → parse_txt_urls/on_Snapshot__71_parse_txt_urls.py ✅
+parse_rss_urls/on_Snapshot__61_parse_rss_urls.py        → parse_rss_urls/on_Snapshot__72_parse_rss_urls.py ✅
+parse_netscape_urls/on_Snapshot__63_parse_netscape_urls.py → parse_netscape_urls/on_Snapshot__73_parse_netscape_urls.py ✅
+parse_jsonl_urls/on_Snapshot__64_parse_jsonl_urls.py    → parse_jsonl_urls/on_Snapshot__74_parse_jsonl_urls.py ✅
+parse_dom_outlinks/on_Snapshot__40_parse_dom_outlinks.js → parse_dom_outlinks/on_Snapshot__75_parse_dom_outlinks.js ✅
+```
+
+## Testing Strategy
+
+### Unit Tests
+- Test hook ordering (00-99)
+- Test step grouping (first digit)
+- Test .bg vs foreground execution
+- Test timeout enforcement
+- Test JSONL parsing
+- Test failure modes & retry_at logic
+
+### Integration Tests
+- Test full Snapshot.run() with mixed hooks
+- Test .bg scripts running beyond step 99
+- Test zombie process cleanup
+- Test graceful SIGTERM handling
+- Test concurrent .bg script coordination
+
+### Performance Tests
+- Measure overhead of per-hook ArchiveResults
+- Test with 50+ concurrent .bg scripts
+- Test filesystem contention with many hooks
+
+## Open Questions
+
+### Q: Should we provide semaphore utilities?
+**A:** No. Keep plugins decoupled. Let them use simple filesystem coordination if needed.
+
+### Q: What happens if ArchiveResult table gets huge?
+**A:** We can delete old successful ArchiveResults periodically, or archive them to cold storage. The important data is in the filesystem outputs.
+
+### Q: Should naturally-exiting .bg scripts still be .bg?
+**A:** Yes. The .bg suffix means "don't block step progression," not "run until step 99." Natural exit is the best case.
+
+## Examples
+
+### Foreground Hook (Sequential DOM Access)
+```python
+#!/usr/bin/env python3
+# archivebox/plugins/screenshot/on_Snapshot__51_screenshot.js
+
+# Runs at step 5, blocks step progression until complete
+# Gets killed if it exceeds SCREENSHOT_TIMEOUT
+
+timeout = get_env_int('SCREENSHOT_TIMEOUT') or get_env_int('TIMEOUT', 60)
+
+try:
+    result = subprocess.run(cmd, capture_output=True, timeout=timeout)
+    if result.returncode == 0:
+        print(json.dumps({
+            "type": "ArchiveResult",
+            "status": "succeeded",
+            "output_str": "screenshot.png"
+        }))
+        sys.exit(0)
+    else:
+        # Temporary failure - will be retried
+        sys.exit(1)
+except subprocess.TimeoutExpired:
+    # Timeout - will be retried
+    sys.exit(1)
+```
+
+### Background Hook (Long-Running Download)
+```python
+#!/usr/bin/env python3
+# archivebox/plugins/ytdlp/on_Snapshot__63_ytdlp.bg.py
+
+# Runs at step 6, doesn't block step progression
+# Gets full YTDLP_TIMEOUT (e.g., 3600s) regardless of when step 99 completes
+
+timeout = get_env_int('YTDLP_TIMEOUT') or get_env_int('TIMEOUT', 3600)
+
+try:
+    result = subprocess.run(['yt-dlp', url], capture_output=True, timeout=timeout)
+    if result.returncode == 0:
+        print(json.dumps({
+            "type": "ArchiveResult",
+            "status": "succeeded",
+            "output_str": "media/"
+        }))
+        sys.exit(0)
+    else:
+        # Hard failure - don't retry
+        print(json.dumps({
+            "type": "ArchiveResult",
+            "status": "failed",
+            "output_str": "Video unavailable"
+        }))
+        sys.exit(0)  # Exit 0 to record the failure
+except subprocess.TimeoutExpired:
+    # Timeout - will be retried
+    sys.exit(1)
+```
+
+### Background Hook with Natural Exit
+```javascript
+#!/usr/bin/env node
+// archivebox/plugins/ssl/on_Snapshot__23_ssl.bg.js
+
+// Sets up listener, captures SSL info, then exits naturally
+// No SIGTERM handler needed - already exits when done
+
+async function main() {
+    const page = await connectToChrome();
+
+    // Set up listener
+    page.on('response', async (response) => {
+        const securityDetails = response.securityDetails();
+        if (securityDetails) {
+            fs.writeFileSync('ssl.json', JSON.stringify(securityDetails));
+        }
+    });
+
+    // Wait for navigation (done by other hook)
+    await waitForNavigation();
+
+    // Emit result
+    console.log(JSON.stringify({
+        type: 'ArchiveResult',
+        status: 'succeeded',
+        output_str: 'ssl.json'
+    }));
+
+    process.exit(0);  // Natural exit - no await indefinitely
+}
+
+main().catch(e => {
+    console.error(`ERROR: ${e.message}`);
+    process.exit(1);  // Will be retried
+});
+```
+
+## Summary
+
+This plan provides:
+- ✅ Clear execution ordering (10 steps, 00-99 numbering)
+- ✅ Async support (.bg suffix)
+- ✅ Independent timeout control per plugin
+- ✅ Flexible failure handling & retry logic
+- ✅ Streaming JSONL output for DB updates
+- ✅ Simple filesystem-based coordination
+- ✅ Backward compatibility during migration
+
+The main implementation work is refactoring `Snapshot.run()` to enforce step ordering and manage .bg script lifecycles. Plugin renumbering is straightforward mechanical work.

archivebox/.flake8

@@ -3,4 +3,4 @@ ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E
 select = F,E9,W
 max-line-length = 130
 max-complexity = 10
-exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv
+exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv,data,data*

archivebox/Architecture.md

@@ -0,0 +1,172 @@
+# ArchiveBox UI
+
+## Page: Getting Started
+
+### What do you want to capture?
+
+- Save some URLs now -> [Add page]
+    - Paste some URLs to archive now
+    - Upload a file containing URLs (bookmarks.html export, RSS.xml feed, markdown file, word doc, PDF, etc.)
+    - Pull in URLs to archive from a remote location (e.g. RSS feed URL, remote TXT file, JSON file, etc.)
+
+- Import URLs from a browser -> [Import page]
+    - Desktop: Get the ArchiveBox Chrome/Firefox extension
+    - Mobile: Get the ArchiveBox iOS App / Android App
+    - Upload a bookmarks.html export file
+    - Upload a browser_history.sqlite3 export file
+
+- Import URLs from a 3rd party bookmarking service -> [Sync page]
+    - Pocket
+    - Pinboard
+    - Instapaper
+    - Wallabag
+    - Zapier, N8N, IFTTT, etc.
+    - Upload a bookmarks.html export, bookmarks.json, RSS, etc. file
+
+- Archive URLs on a schedule -> [Schedule page]
+
+- Archive an entire website -> [Crawl page]
+    - What starting URL/domain?
+    - How deep?
+    - Follow links to external domains?
+    - Follow links to parent URLs?
+    - Maximum number of pages to save?
+    - Maximum number of requests/minute?
+
+- Crawl for URLs with a search engine and save automatically
+    - 
+- Some URLs on a schedule
+- Save an entire website (e.g. `https://example.com`)
+- Save results matching a search query (e.g. "site:example.com")
+- Save a social media feed (e.g. `https://x.com/user/1234567890`)
+
+--------------------------------------------------------------------------------
+
+### Crawls App
+
+- Archive an entire website -> [Crawl page]
+    - What are the starting URLs?
+    - How many hops to follow?
+    - Follow links to external domains?
+    - Follow links to parent URLs?
+    - Maximum number of pages to save?
+    - Maximum number of requests/minute?
+
+
+--------------------------------------------------------------------------------
+
+### Scheduler App
+
+
+- Archive URLs on a schedule -> [Schedule page]
+    - What URL(s)?
+    - How often?
+    - Do you want to discard old snapshots after x amount of time?
+    - Any filter rules?
+    - Want to be notified when changes are detected -> redirect[Alerts app/create new alert(crawl=self)]
+
+
+* Choose Schedule check for new URLs: Schedule.objects.get(pk=xyz)
+    - 1 minute
+    - 5 minutes
+    - 1 hour
+    - 1 day
+
+    * Choose Destination Crawl to archive URLs using : Crawl.objects.get(pk=xyz)
+        - Tags
+        - Persona
+        - Created By ID
+        - Config
+        - Filters
+            - URL patterns to include
+            - URL patterns to exclude
+            - ONLY_NEW= Ignore URLs if already saved once / save URL each time it appears / only save is last save > x time ago
+
+
+--------------------------------------------------------------------------------
+
+### Sources App (For managing sources that ArchiveBox pulls URLs in from)
+
+- Add a new source to pull URLs in from (WIZARD)
+    - Choose URI:
+        - [x] Web UI
+        - [x] CLI
+        - Local filesystem path (directory to monitor for new files containing URLs)
+        - Remote URL (RSS/JSON/XML feed)
+        - Chrome browser profile sync (login using gmail to pull bookmarks/history)
+        - Pocket, Pinboard, Instapaper, Wallabag, etc.
+        - Zapier, N8N, IFTTT, etc.
+        - Local server filesystem path (directory to monitor for new files containing URLs)
+        - Google drive (directory to monitor for new files containing URLs)
+        - Remote server FTP/SFTP/SCP path (directory to monitor for new files containing URLs)
+        - AWS/S3/B2/GCP bucket (directory to monitor for new files containing URLs)
+        - XBrowserSync (login to pull bookmarks)
+    - Choose extractor
+        - auto
+        - RSS
+        - Pocket
+        - etc.
+    - Specify extra Config, e.g.
+        - credentials
+        - extractor tuning options (e.g. verify_ssl, cookies, etc.)
+
+- Provide credentials for the source
+    - API Key
+    - Username / Password
+    - OAuth
+
+--------------------------------------------------------------------------------
+
+### Alerts App
+
+- Create a new alert, choose condition
+    - Get notified when a site goes down (<x% success ratio for Snapshots)
+    - Get notified when a site changes visually more than x% (screenshot diff)
+    - Get notified when a site's text content changes more than x% (text diff)
+    - Get notified when a keyword appears
+    - Get notified when a keyword dissapears
+    - When an AI prompt returns some result
+
+- Choose alert threshold:
+    - any condition is met
+    - all conditions are met
+    - condition is met for x% of URLs
+    - condition is met for x% of time
+
+- Choose how to notify: (List[AlertDestination])
+    - maximum alert frequency
+    - destination type: email / Slack / Webhook / Google Sheet / logfile
+    - destination info:
+        - email address(es)
+        - Slack channel
+        - Webhook URL
+
+- Choose scope:
+    - Choose ArchiveResult scope (extractors): (a query that returns ArchiveResult.objects QuerySet)
+        - All extractors
+        - Only screenshots
+        - Only readability / mercury text
+        - Only video
+        - Only html
+        - Only headers
+
+    - Choose Snapshot scope (URL): (a query that returns Snapshot.objects QuerySet)
+        - All domains
+        - Specific domain
+        - All domains in a tag
+        - All domains in a tag category
+        - All URLs matching a certain regex pattern
+
+    - Choose crawl scope: (a query that returns Crawl.objects QuerySet)
+        - All crawls
+        - Specific crawls
+        - crawls by a certain user
+        - crawls using a certain persona
+
+
+class AlertDestination(models.Model):
+    destination_type: [email, slack, webhook, google_sheet, local logfile, b2/s3/gcp bucket, etc.]
+    maximum_frequency
+    filter_rules
+    credentials
+    alert_template: JINJA2 json/text template that gets populated with alert contents

archivebox/LICENSE

@@ -1 +0,0 @@
-../LICENSE

archivebox/__init__.py

@@ -1,4 +1,116 @@
+#!/usr/bin/env python3
+
+# Welcome to the ArchiveBox source code! Thanks for checking it out!
+#
+# "We are swimming upstream against a great torrent of disorganization.
+# In this, our main obligation is to establish arbitrary enclaves of order and system.
+# It is the greatest possible victory to be, to continue to be, and to have been.
+# No defeat can deprive us of the success of having existed for some moment of time
+# in a universe that seems indifferent to us."
+# --Norber Weiner
+
 __package__ = 'archivebox'
 
+import os
+import sys
+from pathlib import Path
 
-from .monkey_patches import *
+ASCII_LOGO = """
+ █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
+██╔══██╗██╔══██╗██╔════╝██║  ██║██║██║   ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
+███████║██████╔╝██║     ███████║██║██║   ██║█████╗   ██████╔╝██║   ██║ ╚███╔╝ 
+██╔══██║██╔══██╗██║     ██╔══██║██║╚██╗ ██╔╝██╔══╝   ██╔══██╗██║   ██║ ██╔██╗ 
+██║  ██║██║  ██║╚██████╗██║  ██║██║ ╚████╔╝ ███████╗ ██████╔╝╚██████╔╝██╔╝ ██╗
+╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
+"""
+
+PACKAGE_DIR = Path(__file__).resolve().parent
+
+# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models
+# # Migrations reference models like 'machine.Binary' which need to be importable
+# if str(PACKAGE_DIR) not in sys.path:
+#     sys.path.append(str(PACKAGE_DIR))
+
+os.environ['DJANGO_SETTINGS_MODULE'] = 'archivebox.core.settings'
+os.environ['TZ'] = 'UTC'
+
+# detect ArchiveBox user's UID/GID based on data dir ownership
+from .config.permissions import drop_privileges                 # noqa
+drop_privileges()
+
+from .misc.checks import check_not_root, check_not_inside_source_dir, check_io_encoding      # noqa
+check_not_root()
+check_not_inside_source_dir()
+check_io_encoding()
+
+# Install monkey patches for third-party libraries
+from .misc.monkey_patches import *                    # noqa
+
+# Built-in plugin directories
+BUILTIN_PLUGINS_DIR = PACKAGE_DIR / 'plugins'
+USER_PLUGINS_DIR = Path(os.getcwd()) / 'plugins'
+
+# These are kept for backwards compatibility with existing code
+# that checks for plugins. The new hook system uses discover_hooks()
+ALL_PLUGINS = {
+    'builtin': BUILTIN_PLUGINS_DIR,
+    'user': USER_PLUGINS_DIR,
+}
+LOADED_PLUGINS = ALL_PLUGINS
+
+# Setup basic config, constants, paths, and version
+from .config.constants import CONSTANTS                         # noqa
+from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR    # noqa
+from .config.version import VERSION                             # noqa
+
+# Set MACHINE_ID env var so hook scripts can use it
+os.environ.setdefault('MACHINE_ID', CONSTANTS.MACHINE_ID)
+
+__version__ = VERSION
+__author__ = 'ArchiveBox'
+__license__ = 'MIT'
+
+ASCII_ICON = """
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+██████████████████████████████████████████████████████████████████████████████████████████████████ 
+         ██                                                                            ██          
+         ██                                                                            ██        
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                    ████████████████████████████████████                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ██       █████████████████████████ █                    ██          
+         ██                    ████████████████████████████████████                    ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                 ██████████████████████████████████████████                 ██          
+         ██                 ██████████████████████████████████████████                 ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██          
+         ██                                                                            ██        
+         ████████████████████████████████████████████████████████████████████████████████          
+"""

archivebox/__main__.py

@@ -1,11 +1,19 @@
 #!/usr/bin/env python3
-
+"""This is the entrypoint for python -m archivebox ..."""
 __package__ = 'archivebox'
 
+import archivebox      # noqa # make sure monkey patches are applied before anything else
 import sys
 
 from .cli import main
 
+ASCII_LOGO_MINI = r"""
+     _             _     _           ____
+    / \   _ __ ___| |__ (_)_   _____| __ )  _____  __
+   / _ \ | '__/ __| '_ \| \ \ / / _ \  _ \ / _ \ \/ /
+  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  <
+ /_/   \_\_|  \___|_| |_|_| \_/ \___|____/ \___/_/\_\
+"""
 
 if __name__ == '__main__':
     main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/abid_utils/__init__.py

@@ -1 +0,0 @@
-__package__ = 'abid_utils'

archivebox/abid_utils/abid.py

@@ -1,202 +0,0 @@
-from typing import NamedTuple, Any, Union, Optional
-
-import ulid
-import uuid6
-import hashlib
-from urllib.parse import urlparse
-
-from uuid import UUID
-from typeid import TypeID            # type: ignore[import-untyped]
-from datetime import datetime
-
-
-
-ABID_PREFIX_LEN = 4
-ABID_SUFFIX_LEN = 26
-ABID_LEN = 30
-ABID_TS_LEN = 10
-ABID_URI_LEN = 8
-ABID_SUBTYPE_LEN = 2
-ABID_RAND_LEN = 6
-
-DEFAULT_ABID_PREFIX = 'obj_'
-
-# allows people to keep their uris secret on a per-instance basis by changing the salt.
-# the default means everyone can share the same namespace for URI hashes,
-# meaning anyone who has a URI and wants to check if you have it can guess the ABID
-DEFAULT_ABID_URI_SALT = '687c2fff14e3a7780faa5a40c237b19b5b51b089'
-
-
-class ABID(NamedTuple):
-    """
-    e.g. ABID('obj_01HX9FPYTRE4A5CCD901ZYEBQE')
-    """
-    prefix: str            # e.g. obj_
-    ts: str                # e.g. 01HX9FPYTR
-    uri: str               # e.g. E4A5CCD9
-    subtype: str           # e.g. 01
-    rand: str              # e.g. ZYEBQE
-    
-    # salt: str = DEFAULT_ABID_URI_SALT
-
-    def __getattr__(self, attr: str) -> Any:
-        return getattr(self.ulid, attr)
-
-    def __eq__(self, other: Any) -> bool:
-        try:
-            return self.ulid == other.ulid
-        except AttributeError:
-            return NotImplemented
-
-    def __str__(self) -> str:
-        return self.prefix + self.suffix
-
-    def __len__(self) -> int:
-        return len(self.prefix + self.suffix)
-
-    @classmethod
-    def parse(cls, buffer: Union[str, UUID, ulid.ULID, TypeID, 'ABID'], prefix=DEFAULT_ABID_PREFIX) -> 'ABID':
-        assert buffer, f'Attempted to create ABID from null value {buffer}'
-
-        buffer = str(buffer)
-        if '_' in buffer:
-            prefix, suffix = buffer.split('_')
-        else:
-            prefix, suffix = prefix.strip('_'), buffer
-
-        assert len(prefix) == ABID_PREFIX_LEN - 1   # length without trailing _
-        assert len(suffix) == ABID_SUFFIX_LEN, f'Suffix {suffix} from {buffer} was not {ABID_SUFFIX_LEN} chars long'
-
-        return cls(
-            prefix=abid_part_from_prefix(prefix),
-            ts=suffix[0:10].upper(),
-            uri=suffix[10:18].upper(),
-            subtype=suffix[18:20].upper(),
-            rand=suffix[20:26].upper(),
-        )
-    
-    @property
-    def uri_salt(self) -> str:
-        return DEFAULT_ABID_URI_SALT
-
-    @property
-    def suffix(self):
-        return ''.join((self.ts, self.uri, self.subtype, self.rand))
-    
-    @property
-    def ulid(self) -> ulid.ULID:
-        return ulid.parse(self.suffix)
-
-    @property
-    def uuid(self) -> UUID:
-        return self.ulid.uuid
-
-    @property
-    def uuid6(self) -> uuid6.UUID:
-        return uuid6.UUID(hex=self.uuid.hex)
-
-    @property
-    def typeid(self) -> TypeID:
-        return TypeID.from_uuid(prefix=self.prefix.strip('_'), suffix=self.uuid6)
-
-    @property
-    def datetime(self) -> datetime:
-        return self.ulid.timestamp().datetime
-
-
-
-####################################################
-
-
-def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
-    """
-    'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25'
-    """
-    if isinstance(uri, bytes):
-        uri_str: str = uri.decode()
-    else:
-        uri_str = str(uri)
-
-    # only hash the domain part of URLs
-    if '://' in uri_str:
-        try:
-            domain = urlparse(uri_str).netloc
-            if domain:
-                uri_str = domain
-        except AttributeError:
-            pass
-    
-    uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
-
-    return hashlib.sha256(uri_bytes).hexdigest().upper()
-
-def abid_part_from_prefix(prefix: Optional[str]) -> str:
-    """
-    'snp_'
-    """
-    if prefix is None:
-        return 'obj_'
-
-    prefix = prefix.strip('_').lower()
-    assert len(prefix) == 3
-    return prefix + '_'
-
-def abid_part_from_uri(uri: str, salt: str=DEFAULT_ABID_URI_SALT) -> str:
-    """
-    'E4A5CCD9'     # takes first 8 characters of sha256(url)
-    """
-    uri = str(uri)
-    return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
-
-def abid_part_from_ts(ts: Optional[datetime]) -> str:
-    """
-    '01HX9FPYTR'   # produces 10 character Timestamp section of ulid based on added date
-    """
-    return str(ulid.from_timestamp(ts) if ts else ulid.new())[:ABID_TS_LEN]
-
-def abid_part_from_subtype(subtype: str) -> str:
-    """
-    Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
-    Also allows us to change the ulid spec later by putting special sigil values here.
-    """
-    subtype = str(subtype)
-    if len(subtype) == ABID_SUBTYPE_LEN:
-        return subtype
-
-    return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()
-
-def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
-    """
-    'ZYEBQE'   # takes last 6 characters of randomness from existing legacy uuid db field
-    """
-    if rand is None:
-        # if it's None we generate a new random 6 character hex string
-        return str(ulid.new())[-ABID_RAND_LEN:]
-    elif isinstance(rand, UUID):
-        # if it's a uuid we take the last 6 characters of the ULID represation of it
-        return str(ulid.from_uuid(rand))[-ABID_RAND_LEN:]
-    elif isinstance(rand, int):
-        # if it's a BigAutoInteger field we convert it from an int to a 0-padded string
-        rand_str = str(rand)[-ABID_RAND_LEN:]
-        padding_needed = ABID_RAND_LEN - len(rand_str)
-        rand_str = ('0'*padding_needed) + rand_str
-        return rand_str
-
-    # otherwise treat it as a string, take the last 6 characters of it verbatim
-    return str(rand)[-ABID_RAND_LEN:].upper()
-
-
-def abid_from_values(prefix, ts, uri, subtype, rand, salt=DEFAULT_ABID_URI_SALT) -> ABID:
-    """
-    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
-    """
-
-    abid = ABID(
-        prefix=abid_part_from_prefix(prefix),
-        ts=abid_part_from_ts(ts),
-        uri=abid_part_from_uri(uri, salt=salt),
-        subtype=abid_part_from_subtype(subtype),
-        rand=abid_part_from_rand(rand),
-    )
-    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
-    return abid

archivebox/abid_utils/admin.py

@@ -1,18 +0,0 @@
-from django.contrib import admin
-
-
-class ABIDModelAdmin(admin.ModelAdmin):
-    list_display = ('created', 'created_by', 'abid', '__str__')
-    sort_fields = ('created', 'created_by', 'abid', '__str__')
-    readonly_fields = ('abid', 'created', '__str__')
-
-    def get_form(self, request, obj=None, **kwargs):
-        form = super().get_form(request, obj, **kwargs)
-        if 'created_by' in form.base_fields:
-            form.base_fields['created_by'].initial = request.user
-        return form
-
-    # def save_model(self, request, obj, form, change):
-    #     if getattr(obj, 'created_by_id', None) in (None, get_or_create_system_user_pk()):
-    #         obj.created_by = request.user
-    #     obj.save()

archivebox/abid_utils/models.py

@@ -1,344 +0,0 @@
-"""
-This file provides the Django ABIDField and ABIDModel base model to inherit from.
-
-It implements the ArchiveBox ID (ABID) interfaces including abid_values, generate_abid, .abid, .uuid, .id.
-"""
-
-from typing import Any, Dict, Union, List, Set, NamedTuple, cast
-
-from ulid import ULID
-from uuid import uuid4, UUID
-from typeid import TypeID            # type: ignore[import-untyped]
-from datetime import datetime
-from functools import partial
-from charidfield import CharIDField  # type: ignore[import-untyped]
-
-from django.conf import settings
-from django.db import models
-from django.utils import timezone
-from django.db.utils import OperationalError
-from django.contrib.auth import get_user_model
-
-from django_stubs_ext.db.models import TypedModelMeta
-
-from .abid import (
-    ABID,
-    ABID_LEN,
-    ABID_RAND_LEN,
-    ABID_SUFFIX_LEN,
-    DEFAULT_ABID_PREFIX,
-    DEFAULT_ABID_URI_SALT,
-    abid_part_from_prefix,
-    abid_from_values
-)
-
-####################################################
-
-
-# Database Field for typeid/ulid style IDs with a prefix, e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ
-ABIDField = partial(
-    CharIDField,
-    max_length=ABID_LEN,
-    help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
-    default=None,
-    null=True,
-    blank=True,
-    db_index=True,
-    unique=True,
-)
-
-def get_or_create_system_user_pk(username='system'):
-    """Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
-
-    User = get_user_model()
-
-    # if only one user exists total, return that user
-    if User.objects.filter(is_superuser=True).count() == 1:
-        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
-
-    # otherwise, create a dedicated "system" user
-    user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
-    return user.pk
-
-
-class ABIDModel(models.Model):
-    """
-    Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
-    """
-    abid_prefix: str = DEFAULT_ABID_PREFIX  # e.g. 'tag_'
-    abid_ts_src = 'None'                    # e.g. 'self.created'
-    abid_uri_src = 'None'                   # e.g. 'self.uri'
-    abid_subtype_src = 'None'               # e.g. 'self.extractor'
-    abid_rand_src = 'None'                  # e.g. 'self.uuid' or 'self.id'
-
-    # id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
-    # uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
-    abid = ABIDField(prefix=abid_prefix)
-
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    created = models.DateTimeField(auto_now_add=True)
-    modified = models.DateTimeField(auto_now=True)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def save(self, *args: Any, **kwargs: Any) -> None:
-        # when first creating a row, self.ABID is the source of truth
-        # overwrite default prefilled self.id & self.abid with generated self.ABID value
-        if self._state.adding or not self.id:
-            self.id = self.ABID.uuid
-        if self._state.adding or not self.abid:
-            self.abid = str(self.ABID)
-
-        super().save(*args, **kwargs)
-        assert str(self.id) == str(self.ABID.uuid), f'self.id {self.id} does not match self.ABID {self.ABID.uuid}'
-        assert str(self.abid) == str(self.ABID), f'self.abid {self.id} does not match self.ABID {self.ABID.uuid}'
-
-    @property
-    def abid_values(self) -> Dict[str, Any]:
-        return {
-            'prefix': self.abid_prefix,
-            'ts': eval(self.abid_ts_src),
-            'uri': eval(self.abid_uri_src),
-            'subtype': eval(self.abid_subtype_src),
-            'rand': eval(self.abid_rand_src),
-        }
-
-    def generate_abid(self) -> ABID:
-        """
-        Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
-        """
-        prefix, ts, uri, subtype, rand = self.abid_values.values()
-
-        if (not prefix) or prefix == DEFAULT_ABID_PREFIX:
-            suggested_abid = self.__class__.__name__[:3].lower()
-            raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
-
-        if not ts:
-            # default to unix epoch with 00:00:00 UTC
-            ts = datetime.fromtimestamp(0, timezone.utc)     # equivalent to: ts = datetime.utcfromtimestamp(0)
-            print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
-
-        if not uri:
-            uri = str(self)
-            print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
-
-        if not subtype:
-            subtype = self.__class__.__name__
-            print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
-
-        if not rand:
-            rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
-            print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
-
-        abid = abid_from_values(
-            prefix=prefix,
-            ts=ts,
-            uri=uri,
-            subtype=subtype,
-            rand=rand,
-            salt=DEFAULT_ABID_URI_SALT,
-        )
-        assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
-        return abid
-
-    @property
-    def ABID(self) -> ABID:
-        """
-        ULIDParts(timestamp='01HX9FPYTR', url='E4A5CCD9', subtype='00', randomness='ZYEBQE')
-        """
-        
-        # if object is not yet saved to DB, always generate fresh ABID from values
-        if self._state.adding:
-            return self.generate_abid()
-        
-        # otherwise DB is single source of truth, load ABID from existing db pk
-        abid: ABID | None = None
-        try:
-            abid = abid or ABID.parse(self.pk)
-        except Exception:
-            pass
-
-        try:
-            abid = abid or ABID.parse(self.id)
-        except Exception:
-            pass
-
-        try:
-            abid = abid or ABID.parse(cast(str, self.abid))
-        except Exception:
-            pass
-
-        abid = abid or self.generate_abid()
-
-        return abid
-
-    @property
-    def ULID(self) -> ULID:
-        """
-        Get a ulid.ULID representation of the object's ABID.
-        """
-        return self.ABID.ulid
-
-    @property
-    def UUID(self) -> UUID:
-        """
-        Get a uuid.UUID (v4) representation of the object's ABID.
-        """
-        return self.ABID.uuid
-
-    @property
-    def TypeID(self) -> TypeID:
-        """
-        Get a typeid.TypeID (stripe-style) representation of the object's ABID.
-        """
-        return self.ABID.typeid
-
-
-
-####################################################
-
-# Django helpers
-def find_all_abid_prefixes() -> Dict[str, type[models.Model]]:
-    """
-    Return the mapping of all ABID prefixes to their models.
-    e.g. {'tag_': core.models.Tag, 'snp_': core.models.Snapshot, ...}
-    """
-    import django.apps
-    prefix_map = {}
-
-    for model in django.apps.apps.get_models():
-        abid_prefix = getattr(model, 'abid_prefix', None)
-        if abid_prefix:
-            prefix_map[abid_prefix] = model
-    return prefix_map
-
-def find_prefix_for_abid(abid: ABID) -> str:
-    """
-    Find the correct prefix for a given ABID that may have be missing a prefix (slow).
-    e.g. ABID('obj_01BJQMF54D093DXEAWZ6JYRPAQ') -> 'snp_'
-    """
-    # if existing abid prefix is correct, lookup is easy
-    model = find_model_from_abid(abid)
-    if model:
-        assert issubclass(model, ABIDModel)
-        return model.abid_prefix
-
-    # prefix might be obj_ or missing, fuzzy-search to find any object that matches
-    return find_obj_from_abid_rand(abid)[0].abid_prefix
-
-def find_model_from_abid_prefix(prefix: str) -> type[ABIDModel] | None:
-    """
-    Return the Django Model that corresponds to a given ABID prefix.
-    e.g. 'tag_' -> core.models.Tag
-    """
-    prefix = abid_part_from_prefix(prefix)
-
-    import django.apps
-
-    for model in django.apps.apps.get_models():
-        if not issubclass(model, ABIDModel): continue   # skip non-ABID-enabled models
-        if not hasattr(model, 'objects'): continue      # skip abstract models
-
-        if (model.abid_prefix == prefix):
-            return model
-
-    return None
-
-def find_model_from_abid(abid: ABID) -> type[models.Model] | None:
-    """
-    Shortcut for find_model_from_abid_prefix(abid.prefix)
-    """
-    return find_model_from_abid_prefix(abid.prefix)
-
-def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDModel]:
-    """
-    Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
-    e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
-    """
-
-    # convert str to ABID if necessary
-    if isinstance(rand, ABID):
-        abid: ABID = rand
-    else:
-        rand = str(rand)
-        if len(rand) < ABID_SUFFIX_LEN:
-            padding_needed = ABID_SUFFIX_LEN - len(rand)
-            rand = ('0'*padding_needed) + rand
-        abid = ABID.parse(rand)
-
-    import django.apps
-
-    partial_matches: List[ABIDModel] = []
-
-    models_to_try = cast(Set[type[models.Model]], set(filter(bool, (
-        model,
-        find_model_from_abid(abid),
-        *django.apps.apps.get_models(),
-    ))))
-    # print(abid, abid.rand, abid.uuid, models_to_try)
-
-    for model in models_to_try:
-        if not issubclass(model, ABIDModel): continue   # skip Models that arent ABID-enabled
-        if not hasattr(model, 'objects'): continue      # skip abstract Models
-        assert hasattr(model, 'objects')                # force-fix for type hint nit about missing manager https://github.com/typeddjango/django-stubs/issues/1684
-
-        # continue on to try fuzzy searching by randomness portion derived from uuid field
-        try:
-            qs = []
-            if hasattr(model, 'abid'):
-                qs = model.objects.filter(abid__endswith=abid.rand)
-            elif hasattr(model, 'uuid'):
-                qs = model.objects.filter(uuid__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
-            elif hasattr(model, 'id'):
-                # NOTE: this only works on SQLite where every column is a string
-                # other DB backends like postgres dont let you do __endswith if this is a BigAutoInteger field
-                
-                # try to search for uuid=...-2354352
-                # try to search for id=...2354352
-                # try to search for id=2354352
-                qs = model.objects.filter(
-                    models.Q(id__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
-                    | models.Q(id__endswith=abid.rand)
-                    | models.Q(id__startswith=str(int(abid.rand)) if abid.rand.isdigit() else abid.rand)
-                )
-
-            for obj in qs:
-                if obj.generate_abid() == abid:
-                    # found exact match, no need to keep iterating
-                    return [obj]
-                partial_matches.append(obj)
-        except OperationalError as err:
-            print(f'[!] WARNING: Got error while trying to iterate through QuerySet for {model}:', err, '\n')
-
-    return partial_matches
-
-def find_obj_from_abid(abid: ABID, model=None, fuzzy=False) -> Any:
-    """
-    Find an object with a given ABID by filtering possible models for a matching abid/uuid/id (fast).
-    e.g. 'snp_01BJQMF54D093DXEAWZ6JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
-    """
-
-    model = model or find_model_from_abid(abid)
-    assert model, f'Could not find model that could match this ABID type: {abid}'
-
-    try:
-        if hasattr(model, 'abid'):
-            return model.objects.get(abid__endswith=abid.suffix)
-        if hasattr(model, 'uuid'):
-            return model.objects.get(uuid=abid.uuid)
-        return model.objects.get(id=abid.uuid)
-    except model.DoesNotExist:
-        # if the model has an abid field then it shouldve matched, pointless to fuzzy search in that case
-        if hasattr(model, 'abid') or (not fuzzy):
-            raise
-
-    # continue on to try fuzzy searching by randomness portion derived from uuid field
-    match_by_rand = find_obj_from_abid_rand(abid, model=model)
-    if match_by_rand:
-        if match_by_rand[0].abid_prefix != abid.prefix:
-            print(f'[!] WARNING: fetched object {match_by_rand} even though prefix {abid.prefix} doesnt match!', abid, '\n')
-        return match_by_rand
-
-    raise model.DoesNotExist
-

archivebox/api/admin.py

@@ -0,0 +1,68 @@
+__package__ = 'archivebox.api'
+
+from signal_webhooks.admin import WebhookAdmin
+from signal_webhooks.utils import get_webhook_model
+
+from archivebox.base_models.admin import BaseModelAdmin
+
+from archivebox.api.models import APIToken
+
+
+class APITokenAdmin(BaseModelAdmin):
+    list_display = ('created_at', 'id', 'created_by', 'token_redacted', 'expires')
+    sort_fields = ('id', 'created_at', 'created_by', 'expires')
+    readonly_fields = ('created_at', 'modified_at')
+    search_fields = ('id', 'created_by__username', 'token')
+
+    fieldsets = (
+        ('Token', {
+            'fields': ('token', 'expires'),
+            'classes': ('card',),
+        }),
+        ('Owner', {
+            'fields': ('created_by',),
+            'classes': ('card',),
+        }),
+        ('Timestamps', {
+            'fields': ('created_at', 'modified_at'),
+            'classes': ('card',),
+        }),
+    )
+
+    list_filter = ('created_by',)
+    ordering = ['-created_at']
+    list_per_page = 100
+
+
+class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
+    list_display = ('created_at', 'created_by', 'id', *WebhookAdmin.list_display)
+    sort_fields = ('created_at', 'created_by', 'id', 'referenced_model', 'endpoint', 'last_success', 'last_error')
+    readonly_fields = ('created_at', 'modified_at', *WebhookAdmin.readonly_fields)
+
+    fieldsets = (
+        ('Webhook', {
+            'fields': ('name', 'signal', 'referenced_model', 'endpoint'),
+            'classes': ('card', 'wide'),
+        }),
+        ('Authentication', {
+            'fields': ('auth_token',),
+            'classes': ('card',),
+        }),
+        ('Status', {
+            'fields': ('enabled', 'last_success', 'last_error'),
+            'classes': ('card',),
+        }),
+        ('Owner', {
+            'fields': ('created_by',),
+            'classes': ('card',),
+        }),
+        ('Timestamps', {
+            'fields': ('created_at', 'modified_at'),
+            'classes': ('card',),
+        }),
+    )
+
+
+def register_admin(admin_site):
+    admin_site.register(APIToken, APITokenAdmin)
+    admin_site.register(get_webhook_model(), CustomWebhookAdmin)

archivebox/api/apps.py

@@ -3,9 +3,11 @@ __package__ = 'archivebox.api'
 from django.apps import AppConfig
 
 
-
 class APIConfig(AppConfig):
-    name = 'api'
+    name = 'archivebox.api'
+    label = 'api'
 
-    def ready(self):
-        pass
+
+def register_admin(admin_site):
+    from archivebox.api.admin import register_admin
+    register_admin(admin_site)

archivebox/api/auth.py

@@ -1,54 +1,70 @@
 __package__ = 'archivebox.api'
 
 from typing import Optional, cast
+from datetime import timedelta
 
 from django.http import HttpRequest
-from django.contrib.auth import login
+from django.utils import timezone
 from django.contrib.auth import authenticate
 from django.contrib.auth.models import AbstractBaseUser
 
-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser
+from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth
+from ninja.errors import HttpError
+
+
+def get_or_create_api_token(user):
+    from archivebox.api.models import APIToken
+    
+    if user and user.is_superuser:
+        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
+        if api_tokens.exists():
+            # unexpired token exists, use it
+            api_token = api_tokens.last()
+        else:
+            # does not exist, create a new one
+            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
+        
+        assert api_token.is_valid(), f"API token is not valid {api_token}"
+
+        return api_token
+    return None
 
 
 def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
     """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
-    from api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
+    from archivebox.api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
     
     user = None
 
-    submitted_empty_form = token in ('string', '', None)
-    if submitted_empty_form:
-        assert request is not None, 'No request provided for API key authentication'
-        user = request.user       # see if user is authed via django session and use that as the default
-    else:
+    submitted_empty_form = str(token).strip() in ('string', '', 'None', 'null')
+    if not submitted_empty_form:
         try:
             token = APIToken.objects.get(token=token)
             if token.is_valid():
                 user = token.created_by
+                request._api_token = token
         except APIToken.DoesNotExist:
             pass
 
     if not user:
-        print('[❌] Failed to authenticate API user using API Key:', request)
-
-    return None
+        # print('[❌] Failed to authenticate API user using API Key:', request)
+        return None
+    
+    return cast(AbstractBaseUser, user)
 
 def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
     """Given a username and password, check if they are valid and return the corresponding user"""
     user = None
     
     submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
-    if submitted_empty_form:
-        assert request is not None, 'No request provided for API key authentication'
-        user = request.user       # see if user is authed via django session and use that as the default
-    else:
+    if not submitted_empty_form:
         user = authenticate(
             username=username,
             password=password,
         )
 
     if not user:
-        print('[❌] Failed to authenticate API user using API Key:', request)
+        # print('[❌] Failed to authenticate API user using API Key:', request)
         user = None
 
     return cast(AbstractBaseUser | None, user)
@@ -56,28 +72,41 @@ def auth_using_password(username, password, request: Optional[HttpRequest]=None)
 
 ### Base Auth Types
 
+
 class APITokenAuthCheck:
     """The base class for authentication methods that use an api.models.APIToken"""
     def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        user = auth_using_token(
+        request.user = auth_using_token(
             token=key,
             request=request,
         )
-        if user is not None:
-            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
-        return user
+        if request.user and request.user.pk:
+            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
+            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
+            request._api_auth_method = self.__class__.__name__
+
+            if not request.user.is_superuser:
+                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
+        return request.user
+
 
 class UserPassAuthCheck:
     """The base class for authentication methods that use a username & password"""
     def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        user = auth_using_password(
+        request.user = auth_using_password(
             username=username,
             password=password,
             request=request,
         )
-        if user is not None:
-            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
-        return user
+        if request.user and request.user.pk:
+            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
+            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
+            request._api_auth_method = self.__class__.__name__
+
+            if not request.user.is_superuser:
+                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
+
+        return request.user
 
 
 ### Django-Ninja-Provided Auth Methods
@@ -98,13 +127,12 @@ class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
     """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
     pass
 
-
 ### Enabled Auth Methods
 
 API_AUTH_METHODS = [
     HeaderTokenAuth(),
     BearerTokenAuth(),
     QueryParamTokenAuth(), 
-    django_auth_superuser,
+    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
     UsernameAndPasswordAuth(),
 ]

archivebox/api/migrations/0001_initial.py

@@ -1,10 +1,15 @@
-# Generated by Django 4.2.11 on 2024-04-25 04:19
+# Generated by hand on 2025-12-29
+# Creates APIToken and OutboundWebhook tables using raw SQL
 
-import api.models
-from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
-import uuid
+import django.utils.timezone
+from django.conf import settings
+from archivebox.uuid_compat import uuid7
+from archivebox.base_models.models import get_or_create_system_user_pk
+import archivebox.api.models
+import signal_webhooks.fields
+import signal_webhooks.utils
 
 
 class Migration(migrations.Migration):
@@ -12,18 +17,114 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
+        ('auth', '0012_alter_user_first_name_max_length'),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
     ]
 
     operations = [
-        migrations.CreateModel(
-            name='APIToken',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('token', models.CharField(default=api.models.generate_secret_token, max_length=32, unique=True)),
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('expires', models.DateTimeField(blank=True, null=True)),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    sql="""
+                -- Create api_apitoken table
+                CREATE TABLE IF NOT EXISTS api_apitoken (
+                    id TEXT PRIMARY KEY NOT NULL,
+                    created_at DATETIME NOT NULL,
+                    modified_at DATETIME NOT NULL,
+
+                    token VARCHAR(32) NOT NULL UNIQUE,
+                    expires DATETIME,
+
+                    created_by_id INTEGER NOT NULL,
+
+                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
+                );
+                CREATE INDEX IF NOT EXISTS api_apitoken_created_by_id_idx ON api_apitoken(created_by_id);
+                CREATE INDEX IF NOT EXISTS api_apitoken_created_at_idx ON api_apitoken(created_at);
+                CREATE INDEX IF NOT EXISTS api_apitoken_token_idx ON api_apitoken(token);
+
+                -- Create api_outboundwebhook table
+                CREATE TABLE IF NOT EXISTS api_outboundwebhook (
+                    id TEXT PRIMARY KEY NOT NULL,
+                    created_at DATETIME NOT NULL,
+                    modified_at DATETIME NOT NULL,
+
+                    name VARCHAR(255) NOT NULL UNIQUE,
+                    signal VARCHAR(255) NOT NULL,
+                    ref VARCHAR(1024) NOT NULL,
+                    endpoint VARCHAR(2048) NOT NULL,
+                    headers TEXT NOT NULL DEFAULT '{}',
+                    auth_token TEXT NOT NULL DEFAULT '',
+                    enabled BOOLEAN NOT NULL DEFAULT 1,
+                    keep_last_response BOOLEAN NOT NULL DEFAULT 0,
+                    created DATETIME NOT NULL,
+                    updated DATETIME NOT NULL,
+                    last_response TEXT NOT NULL DEFAULT '',
+                    last_success DATETIME,
+                    last_failure DATETIME,
+
+                    created_by_id INTEGER NOT NULL,
+
+                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
+                );
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_by_id_idx ON api_outboundwebhook(created_by_id);
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
+                    """,
+                    reverse_sql="""
+                DROP TABLE IF EXISTS api_outboundwebhook;
+                DROP TABLE IF EXISTS api_apitoken;
+                    """
+                ),
+            ],
+            state_operations=[
+                migrations.CreateModel(
+                    name='APIToken',
+                    fields=[
+                        ('id', models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
+                        ('created_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+                        ('modified_at', models.DateTimeField(auto_now=True)),
+                        ('token', models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
+                        ('expires', models.DateTimeField(blank=True, null=True)),
+                        ('created_by', models.ForeignKey(default=get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                    ],
+                    options={
+                        'verbose_name': 'API Key',
+                        'verbose_name_plural': 'API Keys',
+                        'app_label': 'api',
+                    },
+                ),
+                migrations.CreateModel(
+                    name='OutboundWebhook',
+                    fields=[
+                        ('id', models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
+                        ('created_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+                        ('modified_at', models.DateTimeField(auto_now=True)),
+                        ('name', models.CharField(db_index=True, help_text='Webhook name.', max_length=255, unique=True, verbose_name='name')),
+                        ('signal', models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='Signal the webhook fires to.', max_length=255, verbose_name='signal')),
+                        ('ref', models.CharField(db_index=True, help_text='Dot import notation to the model the webhook is for.', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model')),
+                        ('endpoint', models.URLField(help_text='Target endpoint for this webhook.', max_length=2047, verbose_name='endpoint')),
+                        ('headers', models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers')),
+                        ('auth_token', signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token')),
+                        ('enabled', models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled')),
+                        ('keep_last_response', models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response')),
+                        ('created', models.DateTimeField(auto_now_add=True, help_text='When the webhook was created.', verbose_name='created')),
+                        ('updated', models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated')),
+                        ('last_response', models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response')),
+                        ('last_success', models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success')),
+                        ('last_failure', models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure')),
+                        ('created_by', models.ForeignKey(default=get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                    ],
+                    options={
+                        'verbose_name': 'API Outbound Webhook',
+                        'app_label': 'api',
+                    },
+                ),
+                migrations.AddConstraint(
+                    model_name='outboundwebhook',
+                    constraint=models.UniqueConstraint(fields=['ref', 'endpoint'], name='prevent_duplicate_hooks_api_outboundwebhook'),
+                ),
             ],
         ),
     ]

archivebox/api/migrations/0002_alter_apitoken_options.py

@@ -1,17 +0,0 @@
-# Generated by Django 5.0.4 on 2024-04-26 05:28
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name='apitoken',
-            options={'verbose_name': 'API Key', 'verbose_name_plural': 'API Keys'},
-        ),
-    ]

archivebox/api/migrations/0003_rename_user_apitoken_created_by_apitoken_abid_and_more.py

@@ -1,77 +0,0 @@
-# Generated by Django 5.0.6 on 2024-06-03 01:52
-
-import abid_utils.models
-import charidfield.fields
-import django.db.models.deletion
-import signal_webhooks.fields
-import signal_webhooks.utils
-import uuid
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0002_alter_apitoken_options'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='user',
-            new_name='created_by',
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='abid',
-            field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='apt_', unique=True),
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='modified',
-            field=models.DateTimeField(auto_now=True),
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='uuid',
-            field=models.UUIDField(blank=True, null=True, unique=True),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False),
-        ),
-        migrations.CreateModel(
-            name='OutboundWebhook',
-            fields=[
-                ('name', models.CharField(db_index=True, help_text='Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).', max_length=255, unique=True, verbose_name='name')),
-                ('signal', models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='The type of event the webhook should fire for (e.g. Create, Update, Delete).', max_length=255, verbose_name='signal')),
-                ('ref', models.CharField(db_index=True, help_text='Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model')),
-                ('endpoint', models.URLField(help_text='External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).', max_length=2047, verbose_name='endpoint')),
-                ('headers', models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers')),
-                ('auth_token', signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token')),
-                ('enabled', models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled')),
-                ('keep_last_response', models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response')),
-                ('updated', models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated')),
-                ('last_response', models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response')),
-                ('last_success', models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success')),
-                ('last_failure', models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure')),
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('modified', models.DateTimeField(auto_now=True)),
-                ('id', models.UUIDField(blank=True, null=True, unique=True)),
-                ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
-                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='whk_', unique=True)),
-                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-            options={
-                'verbose_name': 'API Outbound Webhook',
-                'abstract': False,
-            },
-        ),
-        migrations.AddConstraint(
-            model_name='outboundwebhook',
-            constraint=models.UniqueConstraint(fields=('ref', 'endpoint'), name='prevent_duplicate_hooks_api_outboundwebhook'),
-        ),
-    ]

archivebox/api/migrations/0004_alter_apitoken_id_alter_apitoken_uuid.py

@@ -1,24 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 10:44
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0003_rename_user_apitoken_created_by_apitoken_abid_and_more'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='uuid',
-            field=models.UUIDField(blank=True, editable=False, null=True, unique=True),
-        ),
-    ]

archivebox/api/migrations/0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more.py

@@ -1,22 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:40
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0004_alter_apitoken_id_alter_apitoken_uuid'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='apitoken',
-            name='uuid',
-        ),
-        migrations.RemoveField(
-            model_name='outboundwebhook',
-            name='id',
-        ),
-    ]

archivebox/api/migrations/0006_remove_outboundwebhook_uuid_apitoken_id_and_more.py

@@ -1,29 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:43
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more'),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='outboundwebhook',
-            old_name='uuid',
-            new_name='id'
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-    ]

archivebox/api/migrations/0007_alter_apitoken_created_by.py

@@ -1,22 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:52
-
-import abid_utils.models
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0006_remove_outboundwebhook_uuid_apitoken_id_and_more'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created_by',
-            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-    ]

archivebox/api/models.py

@@ -1,122 +1,55 @@
 __package__ = 'archivebox.api'
 
-import uuid
 import secrets
+from archivebox.uuid_compat import uuid7
 from datetime import timedelta
 
 from django.conf import settings
 from django.db import models
 from django.utils import timezone
-
+from django_stubs_ext.db.models import TypedModelMeta
 from signal_webhooks.models import WebhookBase
 
-from django_stubs_ext.db.models import TypedModelMeta
-
-from abid_utils.models import ABIDModel, ABIDField, get_or_create_system_user_pk
-
+from archivebox.base_models.models import get_or_create_system_user_pk
 
 
 def generate_secret_token() -> str:
-    # returns cryptographically secure string with len() == 32
     return secrets.token_hex(16)
 
 
-class APIToken(ABIDModel):
-    """
-    A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox.
-    """
-    # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
-    abid_prefix = 'apt_'
-    abid_ts_src = 'self.created'
-    abid_uri_src = 'self.token'
-    abid_subtype_src = 'self.created_by_id'
-    abid_rand_src = 'self.id'
-
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    abid = ABIDField(prefix=abid_prefix)
-
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    created = models.DateTimeField(auto_now_add=True)
-
+class APIToken(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
     token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
     expires = models.DateTimeField(null=True, blank=True)
 
     class Meta(TypedModelMeta):
+        app_label = 'api'
         verbose_name = "API Key"
         verbose_name_plural = "API Keys"
 
     def __str__(self) -> str:
         return self.token
 
-    def __repr__(self) -> str:
-        return f'<APIToken user={self.created_by.username} token={self.token_redacted}>'
-
-    def __json__(self) -> dict:
-        return {
-            "TYPE":             "APIToken",    
-            "id":               str(self.pk),
-            "abid":             str(self.ABID),
-            "created_by_id":    str(self.created_by_id),
-            "token":            self.token,
-            "created":          self.created.isoformat(),
-            "expires":          self.expires_as_iso8601,
-        }
-
-    @property
-    def expires_as_iso8601(self):
-        """Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
-        expiry_date = self.expires or (timezone.now() + timedelta(days=365 * 100))
-
-        return expiry_date.isoformat()
-    
     @property
     def token_redacted(self):
         return f'************{self.token[-4:]}'
 
     def is_valid(self, for_date=None):
-        for_date = for_date or timezone.now()
-
-        if self.expires and self.expires < for_date:
-            return False
-
-        return True
+        return not self.expires or self.expires >= (for_date or timezone.now())
 
 
-
-
-
-
-# monkey patch django-signals-webhooks to change how it shows up in Admin UI
-
-class OutboundWebhook(ABIDModel, WebhookBase):
-    """
-    Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using:
-        settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
-    """
-    abid_prefix = 'whk_'
-    abid_ts_src = 'self.created'
-    abid_uri_src = 'self.endpoint'
-    abid_subtype_src = 'self.ref'
-    abid_rand_src = 'self.id'
-
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    abid = ABIDField(prefix=abid_prefix)
-
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
-    created = models.DateTimeField(auto_now_add=True)
-    modified = models.DateTimeField(auto_now=True)
-
-    # More fields here: WebhookBase...
-
-    WebhookBase._meta.get_field('name').help_text = (
-        'Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).')
-    WebhookBase._meta.get_field('signal').help_text = (
-        'The type of event the webhook should fire for (e.g. Create, Update, Delete).')
-    WebhookBase._meta.get_field('ref').help_text = (
-        'Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).')
-    WebhookBase._meta.get_field('endpoint').help_text = (
-        'External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).')
+class OutboundWebhook(WebhookBase):
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
 
     class Meta(WebhookBase.Meta):
+        app_label = 'api'
         verbose_name = 'API Outbound Webhook'
 
+    def __str__(self) -> str:
+        return f'[{self.id}] {self.ref} -> {self.endpoint}'

archivebox/api/tests.py

@@ -1,30 +1,30 @@
 __package__ = 'archivebox.api'
 
-from django.test import TestCase
-from ninja.testing import TestClient
+# from django.test import TestCase
+# from ninja.testing import TestClient
 
-from .routes_cli import router
+# from .routes_cli import router
 
-class ArchiveBoxCLIAPITestCase(TestCase):
-    def setUp(self):
-        self.client = TestClient(router)
+# class ArchiveBoxCLIAPITestCase(TestCase):
+#     def setUp(self):
+#         self.client = TestClient(router)
 
-    def test_add_endpoint(self):
-        response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
+#     def test_add_endpoint(self):
+#         response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
 
-    def test_remove_endpoint(self):
-        response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
+#     def test_remove_endpoint(self):
+#         response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
 
-    def test_update_endpoint(self):
-        response = self.client.post("/update", json={})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
+#     def test_update_endpoint(self):
+#         response = self.client.post("/update", json={})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
 
-    def test_list_all_endpoint(self):
-        response = self.client.post("/list_all", json={})
-        self.assertEqual(response.status_code, 200)
-        self.assertTrue(response.json()["success"])
+#     def test_list_all_endpoint(self):
+#         response = self.client.post("/list_all", json={})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])

archivebox/api/v1_api.py

@@ -12,11 +12,13 @@ from ninja import NinjaAPI, Swagger
 
 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
 
-from api.auth import API_AUTH_METHODS
-from ..config import VERSION, COMMIT_HASH
+from archivebox.config import VERSION
+from archivebox.config.version import get_COMMIT_HASH
+
+from archivebox.api.auth import API_AUTH_METHODS
 
 
-COMMIT_HASH = COMMIT_HASH or 'unknown'
+COMMIT_HASH = get_COMMIT_HASH() or 'unknown'
 
 html_description=f'''
 <h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
@@ -35,9 +37,12 @@ html_description=f'''
 
 
 def register_urls(api: NinjaAPI) -> NinjaAPI:
-    api.add_router('/auth/',     'api.v1_auth.router')
-    api.add_router('/core/',     'api.v1_core.router')
-    api.add_router('/cli/',      'api.v1_cli.router')
+    # api.add_router('/auth/',     'archivebox.api.v1_auth.router')
+    api.add_router('/core/',     'archivebox.api.v1_core.router')
+    api.add_router('/crawls/',   'archivebox.api.v1_crawls.router')
+    api.add_router('/cli/',      'archivebox.api.v1_cli.router')
+    api.add_router('/workers/',  'archivebox.api.v1_workers.router')
+    api.add_router('/machine/',  'archivebox.api.v1_machine.router')
     return api
 
 
@@ -52,7 +57,26 @@ class NinjaAPIWithIOCapture(NinjaAPI):
 
                 response = super().create_temporal_response(request)
 
-        print('RESPONDING NOW', response)
+        # Diable caching of API responses entirely
+        response['Cache-Control'] = 'no-store'
+
+        # Add debug stdout and stderr headers to response
+        response['X-ArchiveBox-Stdout'] = str(request.stdout)[200:]
+        response['X-ArchiveBox-Stderr'] = str(request.stderr)[200:]
+        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
+
+        # Add Auth Headers to response
+        api_token = getattr(request, '_api_token', None)
+        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else 'Never'
+
+        response['X-ArchiveBox-Auth-Method'] = getattr(request, '_api_auth_method', None) or 'None'
+        response['X-ArchiveBox-Auth-Expires'] = token_expiry
+        response['X-ArchiveBox-Auth-Token-Id'] = str(api_token.id) if api_token else 'None'
+        response['X-ArchiveBox-Auth-User-Id'] = request.user.pk if request.user.pk else 'None'
+        response['X-ArchiveBox-Auth-User-Username'] = request.user.username if request.user.pk else 'None'
+
+        # import ipdb; ipdb.set_trace()
+        # print('RESPONDING NOW', response)
 
         return response
 
@@ -60,8 +84,7 @@ class NinjaAPIWithIOCapture(NinjaAPI):
 api = NinjaAPIWithIOCapture(
     title='ArchiveBox API',
     description=html_description,
-    version='1.0.0',
-    csrf=False,
+    version=VERSION,
     auth=API_AUTH_METHODS,
     urls_namespace="api-1",
     docs=Swagger(settings={"persistAuthorization": True}),

archivebox/api/v1_auth.py

@@ -3,12 +3,14 @@ __package__ = 'archivebox.api'
 from typing import Optional
 
 from ninja import Router, Schema
+from django.utils import timezone
+from datetime import timedelta
 
-from api.models import APIToken
-from api.auth import auth_using_token, auth_using_password
+from archivebox.api.models import APIToken
+from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token
 
 
-router = Router(tags=['Authentication'])
+router = Router(tags=['Authentication'], auth=None)
 
 
 class PasswordAuthSchema(Schema):
@@ -25,10 +27,9 @@ def get_api_token(request, auth_data: PasswordAuthSchema):
         request=request,
     )
 
-    if user:
-        # TODO: support multiple tokens in the future, for now we just have one per user
-        api_token, created = APIToken.objects.get_or_create(created_by_id=user.pk)
-
+    if user and user.is_superuser:
+        api_token = get_or_create_api_token(user)
+        assert api_token is not None, "Failed to create API token"
         return api_token.__json__()
     
     return {"success": False, "errors": ["Invalid credentials"]}

archivebox/api/v1_cli.py

@@ -1,21 +1,17 @@
 __package__ = 'archivebox.api'
 
+import json
 from typing import List, Dict, Any, Optional
 from enum import Enum
 
 from ninja import Router, Schema
 
-from ..main import (
-    add,
-    remove,
-    update,
-    list_all,
-    schedule,
-)
-from ..util import ansi_to_html
-from ..config import ONLY_NEW
+from archivebox.misc.util import ansi_to_html
+from archivebox.config.common import ARCHIVING_CONFIG
 
 
+# from .auth import API_AUTH_METHODS
+
 # router for API that exposes archivebox cli subcommands as REST endpoints
 router = Router(tags=['ArchiveBox CLI Sub-Commands'])
 
@@ -28,6 +24,7 @@ class CLICommandResponseSchema(Schema):
     success: bool
     errors: List[str]
     result: JSONType
+    result_format: str = 'str'
     stdout: str
     stderr: str
 
@@ -56,17 +53,15 @@ class AddCommandSchema(Schema):
     urls: List[str]
     tag: str = ""
     depth: int = 0
-    update: bool = not ONLY_NEW  # Default to the opposite of ONLY_NEW
-    update_all: bool = False
-    index_only: bool = False
-    overwrite: bool = False
-    init: bool = False
-    extractors: str = ""
     parser: str = "auto"
+    plugins: str = ""
+    update: bool = not ARCHIVING_CONFIG.ONLY_NEW  # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW
+    overwrite: bool = False
+    index_only: bool = False
 
 class UpdateCommandSchema(Schema):
     resume: Optional[float] = 0
-    only_new: bool = ONLY_NEW
+    only_new: bool = ARCHIVING_CONFIG.ONLY_NEW
     index_only: bool = False
     overwrite: bool = False
     after: Optional[float] = 0
@@ -74,7 +69,7 @@ class UpdateCommandSchema(Schema):
     status: Optional[StatusChoices] = StatusChoices.unarchived
     filter_type: Optional[str] = FilterTypeChoices.substring
     filter_patterns: Optional[List[str]] = ['https://example.com']
-    extractors: Optional[str] = ""
+    plugins: Optional[str] = ""
 
 class ScheduleCommandSchema(Schema):
     import_path: Optional[str] = None
@@ -83,19 +78,19 @@ class ScheduleCommandSchema(Schema):
     tag: str = ''
     depth: int = 0
     overwrite: bool = False
-    update: bool = not ONLY_NEW
+    update: bool = not ARCHIVING_CONFIG.ONLY_NEW
     clear: bool = False
 
 class ListCommandSchema(Schema):
     filter_patterns: Optional[List[str]] = ['https://example.com']
     filter_type: str = FilterTypeChoices.substring
-    status: Optional[StatusChoices] = StatusChoices.indexed
+    status: StatusChoices = StatusChoices.indexed
     after: Optional[float] = 0
     before: Optional[float] = 999999999999999
-    sort: str = 'added'
+    sort: str = 'bookmarked_at'
     as_json: bool = True
     as_html: bool = False
-    as_csv: str | bool = 'timestamp,url'
+    as_csv: str | None = 'timestamp,url'
     with_headers: bool = False
 
 class RemoveCommandSchema(Schema):
@@ -111,17 +106,19 @@ class RemoveCommandSchema(Schema):
 
 @router.post("/add", response=CLICommandResponseSchema, summary='archivebox add [args] [urls]')
 def cli_add(request, args: AddCommandSchema):
+    from archivebox.cli.archivebox_add import add
+
     result = add(
         urls=args.urls,
         tag=args.tag,
         depth=args.depth,
         update=args.update,
-        update_all=args.update_all,
         index_only=args.index_only,
         overwrite=args.overwrite,
-        init=args.init,
-        extractors=args.extractors,
+        plugins=args.plugins,
         parser=args.parser,
+        bg=True,  # Always run in background for API calls
+        created_by_id=request.user.pk,
     )
 
     return {
@@ -135,6 +132,8 @@ def cli_add(request, args: AddCommandSchema):
 
 @router.post("/update", response=CLICommandResponseSchema, summary='archivebox update [args] [filter_patterns]')
 def cli_update(request, args: UpdateCommandSchema):
+    from archivebox.cli.archivebox_update import update
+    
     result = update(
         resume=args.resume,
         only_new=args.only_new,
@@ -145,7 +144,7 @@ def cli_update(request, args: UpdateCommandSchema):
         status=args.status,
         filter_type=args.filter_type,
         filter_patterns=args.filter_patterns,
-        extractors=args.extractors,
+        plugins=args.plugins,
     )
     return {
         "success": True,
@@ -158,6 +157,8 @@ def cli_update(request, args: UpdateCommandSchema):
 
 @router.post("/schedule", response=CLICommandResponseSchema, summary='archivebox schedule [args] [import_path]')
 def cli_schedule(request, args: ScheduleCommandSchema):
+    from archivebox.cli.archivebox_schedule import schedule
+    
     result = schedule(
         import_path=args.import_path,
         add=args.add,
@@ -180,9 +181,11 @@ def cli_schedule(request, args: ScheduleCommandSchema):
 
 
 
-@router.post("/list", response=CLICommandResponseSchema, summary='archivebox list [args] [filter_patterns]')
-def cli_list(request, args: ListCommandSchema):
-    result = list_all(
+@router.post("/search", response=CLICommandResponseSchema, summary='archivebox search [args] [filter_patterns]')
+def cli_search(request, args: ListCommandSchema):
+    from archivebox.cli.archivebox_search import search
+    
+    result = search(
         filter_patterns=args.filter_patterns,
         filter_type=args.filter_type,
         status=args.status,
@@ -198,6 +201,7 @@ def cli_list(request, args: ListCommandSchema):
     result_format = 'txt'
     if args.as_json:
         result_format = "json"
+        result = json.loads(result)
     elif args.as_html:
         result_format = "html"
     elif args.as_csv:
@@ -216,6 +220,8 @@ def cli_list(request, args: ListCommandSchema):
 
 @router.post("/remove", response=CLICommandResponseSchema, summary='archivebox remove [args] [filter_patterns]')
 def cli_remove(request, args: RemoveCommandSchema):
+    from archivebox.cli.archivebox_remove import remove
+    
     result = remove(
         yes=True,            # no way to interactively ask for confirmation via API, so we force yes
         delete=args.delete,

archivebox/api/v1_core.py

@@ -6,27 +6,27 @@ from typing import List, Optional, Union, Any
 from datetime import datetime
 
 from django.db.models import Q
-from django.shortcuts import get_object_or_404
 from django.core.exceptions import ValidationError
 from django.contrib.auth import get_user_model
+from django.shortcuts import redirect
 
 from ninja import Router, Schema, FilterSchema, Field, Query
 from ninja.pagination import paginate, PaginationBase
+from ninja.errors import HttpError
+
+from archivebox.core.models import Snapshot, ArchiveResult, Tag
+from archivebox.api.v1_crawls import CrawlSchema
 
-from core.models import Snapshot, ArchiveResult, Tag
-from abid_utils.abid import ABID
 
 router = Router(tags=['Core Models'])
 
 
-
 class CustomPagination(PaginationBase):
     class Input(Schema):
         limit: int = 200
         offset: int = 0
         page: int = 0
 
-
     class Output(Schema):
         total_items: int
         total_pages: int
@@ -56,184 +56,126 @@ class CustomPagination(PaginationBase):
 
 ### ArchiveResult #########################################################################
 
-class ArchiveResultSchema(Schema):
+class MinimalArchiveResultSchema(Schema):
     TYPE: str = 'core.models.ArchiveResult'
-
     id: UUID
-    old_id: int
-    abid: str
-
-    modified: datetime
-    created: datetime
+    created_at: datetime | None
+    modified_at: datetime | None
     created_by_id: str
     created_by_username: str
-
-    snapshot_abid: str
-    snapshot_timestamp: str
-    snapshot_url: str
-    snapshot_tags: str
-
-    extractor: str
-    cmd_version: Optional[str]
-    cmd: List[str]
-    pwd: str
     status: str
-    output: str
+    retry_at: datetime | None
+    plugin: str
+    hook_name: str
+    process_id: UUID | None
+    cmd_version: str | None
+    cmd: list[str] | None
+    pwd: str | None
+    output_str: str
+    output_json: dict | None
+    output_files: dict | None
+    output_size: int
+    output_mimetypes: str
+    start_ts: datetime | None
+    end_ts: datetime | None
 
     @staticmethod
     def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-    
-    @staticmethod
-    def resolve_created_by_username(obj):
-        User = get_user_model()
-        return User.objects.get(id=obj.created_by_id).username
+        return str(obj.created_by.pk)
 
     @staticmethod
-    def resolve_pk(obj):
-        return str(obj.pk)
+    def resolve_created_by_username(obj) -> str:
+        return obj.created_by.username
 
-    @staticmethod
-    def resolve_uuid(obj):
-        return str(obj.uuid)
 
-    @staticmethod
-    def resolve_abid(obj):
-        return str(obj.ABID)
-
-    @staticmethod
-    def resolve_created(obj):
-        return obj.start_ts
+class ArchiveResultSchema(MinimalArchiveResultSchema):
+    TYPE: str = 'core.models.ArchiveResult'
+    snapshot_id: UUID
+    snapshot_timestamp: str
+    snapshot_url: str
+    snapshot_tags: List[str]
 
     @staticmethod
     def resolve_snapshot_timestamp(obj):
         return obj.snapshot.timestamp
-    
+
     @staticmethod
     def resolve_snapshot_url(obj):
         return obj.snapshot.url
 
     @staticmethod
-    def resolve_snapshot_abid(obj):
-        return str(obj.snapshot.ABID)
+    def resolve_snapshot_id(obj):
+        return obj.snapshot_id
 
     @staticmethod
     def resolve_snapshot_tags(obj):
-        return obj.snapshot.tags_str()
+        return sorted(tag.name for tag in obj.snapshot.tags.all())
 
 
 class ArchiveResultFilterSchema(FilterSchema):
-    id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
-
-    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
-    snapshot_id: Optional[str] = Field(None, q=['snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+    id: Optional[str] = Field(None, q=['id__startswith', 'snapshot__id__startswith', 'snapshot__timestamp__startswith'])
+    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'plugin', 'output_str__icontains', 'id__startswith', 'snapshot__id__startswith', 'snapshot__timestamp__startswith'])
+    snapshot_id: Optional[str] = Field(None, q=['snapshot__id__startswith', 'snapshot__timestamp__startswith'])
     snapshot_url: Optional[str] = Field(None, q='snapshot__url__icontains')
     snapshot_tag: Optional[str] = Field(None, q='snapshot__tags__name__icontains')
-    
     status: Optional[str] = Field(None, q='status')
-    output: Optional[str] = Field(None, q='output__icontains')
-    extractor: Optional[str] = Field(None, q='extractor__icontains')
+    output_str: Optional[str] = Field(None, q='output_str__icontains')
+    plugin: Optional[str] = Field(None, q='plugin__icontains')
+    hook_name: Optional[str] = Field(None, q='hook_name__icontains')
+    process_id: Optional[str] = Field(None, q='process__id__startswith')
     cmd: Optional[str] = Field(None, q='cmd__0__icontains')
     pwd: Optional[str] = Field(None, q='pwd__icontains')
     cmd_version: Optional[str] = Field(None, q='cmd_version')
-
-    created: Optional[datetime] = Field(None, q='updated')
-    created__gte: Optional[datetime] = Field(None, q='updated__gte')
-    created__lt: Optional[datetime] = Field(None, q='updated__lt')
+    created_at: Optional[datetime] = Field(None, q='created_at')
+    created_at__gte: Optional[datetime] = Field(None, q='created_at__gte')
+    created_at__lt: Optional[datetime] = Field(None, q='created_at__lt')
 
 
 @router.get("/archiveresults", response=List[ArchiveResultSchema], url_name="get_archiveresult")
 @paginate(CustomPagination)
 def get_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...)):
     """List all ArchiveResult entries matching these filters."""
-    qs = ArchiveResult.objects.all()
-    results = filters.filter(qs).distinct()
-    return results
+    return filters.filter(ArchiveResult.objects.all()).distinct()
 
 
 @router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
 def get_archiveresult(request, archiveresult_id: str):
-    """Get a specific ArchiveResult by pk, abid, or old_id."""
-    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id) | Q(old_id__icontains=archiveresult_id))
-
-
-# @router.post("/archiveresult", response=ArchiveResultSchema)
-# def create_archiveresult(request, payload: ArchiveResultSchema):
-#     archiveresult = ArchiveResult.objects.create(**payload.dict())
-#     return archiveresult
-#
-# @router.put("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema)
-# def update_archiveresult(request, archiveresult_id: str, payload: ArchiveResultSchema):
-#     archiveresult = get_object_or_404(ArchiveResult, id=archiveresult_id)
-#   
-#     for attr, value in payload.dict().items():
-#         setattr(archiveresult, attr, value)
-#     archiveresult.save()
-#
-#     return archiveresult
-#
-# @router.delete("/archiveresult/{archiveresult_id}")
-# def delete_archiveresult(request, archiveresult_id: str):
-#     archiveresult = get_object_or_404(ArchiveResult, id=archiveresult_id)
-#     archiveresult.delete()
-#     return {"success": True}
-
-
-
+    """Get a specific ArchiveResult by id."""
+    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id))
 
 
 ### Snapshot #########################################################################
 
-
 class SnapshotSchema(Schema):
     TYPE: str = 'core.models.Snapshot'
-
     id: UUID
-    old_id: UUID
-    abid: str
-
-    modified: datetime
-    created: datetime
     created_by_id: str
     created_by_username: str
-
+    created_at: datetime
+    modified_at: datetime
+    status: str
+    retry_at: datetime | None
+    bookmarked_at: datetime
+    downloaded_at: Optional[datetime]
     url: str
-    tags: str
+    tags: List[str]
     title: Optional[str]
     timestamp: str
     archive_path: str
-
-    bookmarked: datetime
-    added: datetime
-    updated: Optional[datetime]
-
     num_archiveresults: int
-    archiveresults: List[ArchiveResultSchema]
+    archiveresults: List[MinimalArchiveResultSchema]
 
     @staticmethod
     def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-    
+        return str(obj.created_by.pk)
+
     @staticmethod
     def resolve_created_by_username(obj):
-        User = get_user_model()
-        return User.objects.get(id=obj.created_by_id).username
-
-    @staticmethod
-    def resolve_pk(obj):
-        return str(obj.pk)
-
-    @staticmethod
-    def resolve_uuid(obj):
-        return str(obj.uuid)
-
-    @staticmethod
-    def resolve_abid(obj):
-        return str(obj.ABID)
+        return obj.created_by.username
 
     @staticmethod
     def resolve_tags(obj):
-        return obj.tags_str()
+        return sorted(tag.name for tag in obj.tags.all())
 
     @staticmethod
     def resolve_num_archiveresults(obj, context):
@@ -247,119 +189,65 @@ class SnapshotSchema(Schema):
 
 
 class SnapshotFilterSchema(FilterSchema):
-    id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
-
-    old_id: Optional[str] = Field(None, q='old_id__icontains')
-    abid: Optional[str] = Field(None, q='abid__icontains')
-
-    created_by_id: str = Field(None, q='created_by_id')
-    created_by_username: str = Field(None, q='created_by__username__icontains')
-
-    created__gte: datetime = Field(None, q='created__gte')
-    created__lt: datetime = Field(None, q='created__lt')
-    created: datetime = Field(None, q='created')
-    modified: datetime = Field(None, q='modified')
-    modified__gte: datetime = Field(None, q='modified__gte')
-    modified__lt: datetime = Field(None, q='modified__lt')
-
-    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
+    id: Optional[str] = Field(None, q=['id__icontains', 'timestamp__startswith'])
+    created_by_id: str = Field(None, q='crawl__created_by_id')
+    created_by_username: str = Field(None, q='crawl__created_by__username__icontains')
+    created_at__gte: datetime = Field(None, q='created_at__gte')
+    created_at__lt: datetime = Field(None, q='created_at__lt')
+    created_at: datetime = Field(None, q='created_at')
+    modified_at: datetime = Field(None, q='modified_at')
+    modified_at__gte: datetime = Field(None, q='modified_at__gte')
+    modified_at__lt: datetime = Field(None, q='modified_at__lt')
+    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'timestamp__startswith'])
     url: Optional[str] = Field(None, q='url')
     tag: Optional[str] = Field(None, q='tags__name')
     title: Optional[str] = Field(None, q='title__icontains')
     timestamp: Optional[str] = Field(None, q='timestamp__startswith')
-    
-    added__gte: Optional[datetime] = Field(None, q='added__gte')
-    added__lt: Optional[datetime] = Field(None, q='added__lt')
-
+    bookmarked_at__gte: Optional[datetime] = Field(None, q='bookmarked_at__gte')
+    bookmarked_at__lt: Optional[datetime] = Field(None, q='bookmarked_at__lt')
 
 
 @router.get("/snapshots", response=List[SnapshotSchema], url_name="get_snapshots")
 @paginate(CustomPagination)
-def get_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_archiveresults: bool=False):
+def get_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_archiveresults: bool = False):
     """List all Snapshot entries matching these filters."""
     request.with_archiveresults = with_archiveresults
+    return filters.filter(Snapshot.objects.all()).distinct()
 
-    qs = Snapshot.objects.all()
-    results = filters.filter(qs).distinct()
-    return results
 
 @router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
-def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
-    """Get a specific Snapshot by abid, uuid, or pk."""
+def get_snapshot(request, snapshot_id: str, with_archiveresults: bool = True):
+    """Get a specific Snapshot by id."""
     request.with_archiveresults = with_archiveresults
-    snapshot = None
     try:
-        snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(old_id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
+        return Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
-        pass
-
-    try:
-        snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
-    except Snapshot.DoesNotExist:
-        pass
-
-    if not snapshot:
-        raise Snapshot.DoesNotExist
-
-    return snapshot
-
-
-# @router.post("/snapshot", response=SnapshotSchema)
-# def create_snapshot(request, payload: SnapshotSchema):
-#     snapshot = Snapshot.objects.create(**payload.dict())
-#     return snapshot
-#
-# @router.put("/snapshot/{snapshot_id}", response=SnapshotSchema)
-# def update_snapshot(request, snapshot_id: str, payload: SnapshotSchema):
-#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
-#
-#     for attr, value in payload.dict().items():
-#         setattr(snapshot, attr, value)
-#     snapshot.save()
-#
-#     return snapshot
-#
-# @router.delete("/snapshot/{snapshot_id}")
-# def delete_snapshot(request, snapshot_id: str):
-#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
-#     snapshot.delete()
-#     return {"success": True}
-
+        return Snapshot.objects.get(Q(id__icontains=snapshot_id))
 
 
 ### Tag #########################################################################
 
-
 class TagSchema(Schema):
     TYPE: str = 'core.models.Tag'
-
     id: UUID
-    old_id: str
-    abid: str
-
-    modified: datetime
-    created: datetime
+    modified_at: datetime
+    created_at: datetime
     created_by_id: str
     created_by_username: str
-
     name: str
     slug: str
     num_snapshots: int
     snapshots: List[SnapshotSchema]
 
-    @staticmethod
-    def resolve_old_id(obj):
-        return str(obj.old_id)
-
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
-    
+
     @staticmethod
     def resolve_created_by_username(obj):
         User = get_user_model()
         return User.objects.get(id=obj.created_by_id).username
-    
+
     @staticmethod
     def resolve_num_snapshots(obj, context):
         return obj.snapshot_set.all().distinct().count()
@@ -370,6 +258,7 @@ class TagSchema(Schema):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
+
 @router.get("/tags", response=List[TagSchema], url_name="get_tags")
 @paginate(CustomPagination)
 def get_tags(request):
@@ -377,48 +266,194 @@ def get_tags(request):
     request.with_archiveresults = False
     return Tag.objects.all().distinct()
 
+
 @router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
-def get_tag(request, tag_id: str, with_snapshots: bool=True):
+def get_tag(request, tag_id: str, with_snapshots: bool = True):
     request.with_snapshots = with_snapshots
     request.with_archiveresults = False
-    tag = None
     try:
-        tag = tag or Tag.objects.get(old_id__icontains=tag_id)
-    except (Tag.DoesNotExist, ValidationError, ValueError):
-        pass
-
-    try:
-        tag = Tag.objects.get(abid__icontains=tag_id)
+        return Tag.objects.get(id__icontains=tag_id)
     except (Tag.DoesNotExist, ValidationError):
-        pass
-
-    try:
-        tag = tag or Tag.objects.get(id__icontains=tag_id)
-    except (Tag.DoesNotExist, ValidationError):
-        pass
-    return tag
+        return Tag.objects.get(slug__icontains=tag_id)
 
 
-
-@router.get("/any/{abid}", response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema], url_name="get_any")
-def get_any(request, abid: str):
+@router.get("/any/{id}", response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema, CrawlSchema], url_name="get_any", summary="Get any object by its ID")
+def get_any(request, id: str):
+    """Get any object by its ID (e.g. snapshot, archiveresult, tag, crawl, etc.)."""
     request.with_snapshots = False
     request.with_archiveresults = False
 
-    response = None
+    for getter in [get_snapshot, get_archiveresult, get_tag]:
+        try:
+            response = getter(request, id)
+            if response:
+                return redirect(f"/api/v1/{response._meta.app_label}/{response._meta.model_name}/{response.id}?{request.META['QUERY_STRING']}")
+        except Exception:
+            pass
+
     try:
-        response = response or get_snapshot(request, abid)
+        from archivebox.api.v1_crawls import get_crawl
+        response = get_crawl(request, id)
+        if response:
+            return redirect(f"/api/v1/{response._meta.app_label}/{response._meta.model_name}/{response.id}?{request.META['QUERY_STRING']}")
     except Exception:
         pass
 
-    try:
-        response = response or get_archiveresult(request, abid)
-    except Exception:
-        pass
+    raise HttpError(404, 'Object with given ID not found')
 
-    try:
-        response = response or get_tag(request, abid)
-    except Exception:
-        pass
 
-    return response
+### Tag Editor API Endpoints #########################################################################
+
+class TagAutocompleteSchema(Schema):
+    tags: List[dict]
+
+
+class TagCreateSchema(Schema):
+    name: str
+
+
+class TagCreateResponseSchema(Schema):
+    success: bool
+    tag_id: int
+    tag_name: str
+    created: bool
+
+
+class TagSnapshotRequestSchema(Schema):
+    snapshot_id: str
+    tag_name: Optional[str] = None
+    tag_id: Optional[int] = None
+
+
+class TagSnapshotResponseSchema(Schema):
+    success: bool
+    tag_id: int
+    tag_name: str
+
+
+@router.get("/tags/autocomplete/", response=TagAutocompleteSchema, url_name="tags_autocomplete")
+def tags_autocomplete(request, q: str = ""):
+    """Return tags matching the query for autocomplete."""
+    if not q:
+        # Return all tags if no query (limited to 50)
+        tags = Tag.objects.all().order_by('name')[:50]
+    else:
+        tags = Tag.objects.filter(name__icontains=q).order_by('name')[:20]
+
+    return {
+        'tags': [{'id': tag.pk, 'name': tag.name, 'slug': tag.slug} for tag in tags]
+    }
+
+
+@router.post("/tags/create/", response=TagCreateResponseSchema, url_name="tags_create")
+def tags_create(request, data: TagCreateSchema):
+    """Create a new tag or return existing one."""
+    name = data.name.strip()
+    if not name:
+        raise HttpError(400, 'Tag name is required')
+
+    tag, created = Tag.objects.get_or_create(
+        name__iexact=name,
+        defaults={
+            'name': name,
+            'created_by': request.user if request.user.is_authenticated else None,
+        }
+    )
+
+    # If found by case-insensitive match, use that tag
+    if not created:
+        tag = Tag.objects.filter(name__iexact=name).first()
+
+    return {
+        'success': True,
+        'tag_id': tag.pk,
+        'tag_name': tag.name,
+        'created': created,
+    }
+
+
+@router.post("/tags/add-to-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_add_to_snapshot")
+def tags_add_to_snapshot(request, data: TagSnapshotRequestSchema):
+    """Add a tag to a snapshot. Creates the tag if it doesn't exist."""
+    # Get the snapshot
+    try:
+        snapshot = Snapshot.objects.get(
+            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id)
+        )
+    except Snapshot.DoesNotExist:
+        raise HttpError(404, 'Snapshot not found')
+    except Snapshot.MultipleObjectsReturned:
+        snapshot = Snapshot.objects.filter(
+            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id)
+        ).first()
+
+    # Get or create the tag
+    if data.tag_name:
+        name = data.tag_name.strip()
+        if not name:
+            raise HttpError(400, 'Tag name is required')
+
+        tag, _ = Tag.objects.get_or_create(
+            name__iexact=name,
+            defaults={
+                'name': name,
+                'created_by': request.user if request.user.is_authenticated else None,
+            }
+        )
+        # If found by case-insensitive match, use that tag
+        tag = Tag.objects.filter(name__iexact=name).first() or tag
+    elif data.tag_id:
+        try:
+            tag = Tag.objects.get(pk=data.tag_id)
+        except Tag.DoesNotExist:
+            raise HttpError(404, 'Tag not found')
+    else:
+        raise HttpError(400, 'Either tag_name or tag_id is required')
+
+    # Add the tag to the snapshot
+    snapshot.tags.add(tag)
+
+    return {
+        'success': True,
+        'tag_id': tag.pk,
+        'tag_name': tag.name,
+    }
+
+
+@router.post("/tags/remove-from-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_remove_from_snapshot")
+def tags_remove_from_snapshot(request, data: TagSnapshotRequestSchema):
+    """Remove a tag from a snapshot."""
+    # Get the snapshot
+    try:
+        snapshot = Snapshot.objects.get(
+            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id)
+        )
+    except Snapshot.DoesNotExist:
+        raise HttpError(404, 'Snapshot not found')
+    except Snapshot.MultipleObjectsReturned:
+        snapshot = Snapshot.objects.filter(
+            Q(id__startswith=data.snapshot_id) | Q(timestamp__startswith=data.snapshot_id)
+        ).first()
+
+    # Get the tag
+    if data.tag_id:
+        try:
+            tag = Tag.objects.get(pk=data.tag_id)
+        except Tag.DoesNotExist:
+            raise HttpError(404, 'Tag not found')
+    elif data.tag_name:
+        try:
+            tag = Tag.objects.get(name__iexact=data.tag_name.strip())
+        except Tag.DoesNotExist:
+            raise HttpError(404, 'Tag not found')
+    else:
+        raise HttpError(400, 'Either tag_name or tag_id is required')
+
+    # Remove the tag from the snapshot
+    snapshot.tags.remove(tag)
+
+    return {
+        'success': True,
+        'tag_id': tag.pk,
+        'tag_name': tag.name,
+    }

archivebox/api/v1_crawls.py

@@ -0,0 +1,81 @@
+__package__ = 'archivebox.api'
+
+from uuid import UUID
+from typing import List
+from datetime import datetime
+
+from django.db.models import Q
+from django.contrib.auth import get_user_model
+
+from ninja import Router, Schema
+
+from archivebox.core.models import Snapshot
+from archivebox.crawls.models import Crawl
+
+from .auth import API_AUTH_METHODS
+
+router = Router(tags=['Crawl Models'], auth=API_AUTH_METHODS)
+
+
+class CrawlSchema(Schema):
+    TYPE: str = 'crawls.models.Crawl'
+
+    id: UUID
+
+    modified_at: datetime
+    created_at: datetime
+    created_by_id: str
+    created_by_username: str
+
+    status: str
+    retry_at: datetime | None
+
+    urls: str
+    extractor: str
+    max_depth: int
+    tags_str: str
+    config: dict
+
+    # snapshots: List[SnapshotSchema]
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+
+    @staticmethod
+    def resolve_created_by_username(obj):
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username
+
+    @staticmethod
+    def resolve_snapshots(obj, context):
+        if context['request'].with_snapshots:
+            return obj.snapshot_set.all().distinct()
+        return Snapshot.objects.none()
+
+
+@router.get("/crawls", response=List[CrawlSchema], url_name="get_crawls")
+def get_crawls(request):
+    return Crawl.objects.all().distinct()
+
+@router.get("/crawl/{crawl_id}", response=CrawlSchema | str, url_name="get_crawl")
+def get_crawl(request, crawl_id: str, as_rss: bool=False, with_snapshots: bool=False, with_archiveresults: bool=False):
+    """Get a specific Crawl by id."""
+    request.with_snapshots = with_snapshots
+    request.with_archiveresults = with_archiveresults
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    
+    if crawl and as_rss:
+        # return snapshots as XML rss feed
+        urls = [
+            {'url': snapshot.url, 'title': snapshot.title, 'bookmarked_at': snapshot.bookmarked_at, 'tags': snapshot.tags_str}
+            for snapshot in crawl.snapshot_set.all()
+        ]
+        xml = '<rss version="2.0"><channel>'
+        for url in urls:
+            xml += f'<item><url>{url["url"]}</url><title>{url["title"]}</title><bookmarked_at>{url["bookmarked_at"]}</bookmarked_at><tags>{url["tags"]}</tags></item>'
+        xml += '</channel></rss>'
+        return xml
+    
+    return crawl
+

archivebox/api/v1_machine.py

@@ -0,0 +1,150 @@
+__package__ = 'archivebox.api'
+
+from uuid import UUID
+from typing import List, Optional
+from datetime import datetime
+
+from ninja import Router, Schema, FilterSchema, Field, Query
+from ninja.pagination import paginate
+
+from archivebox.api.v1_core import CustomPagination
+
+
+router = Router(tags=['Machine and Dependencies'])
+
+
+# ============================================================================
+# Machine Schemas
+# ============================================================================
+
+class MachineSchema(Schema):
+    """Schema for Machine model."""
+    TYPE: str = 'machine.Machine'
+    id: UUID
+    created_at: datetime
+    modified_at: datetime
+    guid: str
+    hostname: str
+    hw_in_docker: bool
+    hw_in_vm: bool
+    hw_manufacturer: str
+    hw_product: str
+    hw_uuid: str
+    os_arch: str
+    os_family: str
+    os_platform: str
+    os_release: str
+    os_kernel: str
+    stats: dict
+    num_uses_succeeded: int
+    num_uses_failed: int
+
+
+class MachineFilterSchema(FilterSchema):
+    id: Optional[str] = Field(None, q='id__startswith')
+    hostname: Optional[str] = Field(None, q='hostname__icontains')
+    os_platform: Optional[str] = Field(None, q='os_platform__icontains')
+    os_arch: Optional[str] = Field(None, q='os_arch')
+    hw_in_docker: Optional[bool] = Field(None, q='hw_in_docker')
+    hw_in_vm: Optional[bool] = Field(None, q='hw_in_vm')
+
+
+# ============================================================================
+    bin_providers: Optional[str] = Field(None, q='bin_providers__icontains')
+
+
+# ============================================================================
+# Binary Schemas
+# ============================================================================
+
+class BinarySchema(Schema):
+    """Schema for Binary model."""
+    TYPE: str = 'machine.Binary'
+    id: UUID
+    created_at: datetime
+    modified_at: datetime
+    machine_id: UUID
+    machine_hostname: str
+    name: str
+    binproviders: str
+    binprovider: str
+    abspath: str
+    version: str
+    sha256: str
+    status: str
+    is_valid: bool
+    num_uses_succeeded: int
+    num_uses_failed: int
+
+    @staticmethod
+    def resolve_machine_hostname(obj) -> str:
+        return obj.machine.hostname
+
+    @staticmethod
+    def resolve_is_valid(obj) -> bool:
+        return obj.is_valid
+
+
+class BinaryFilterSchema(FilterSchema):
+    id: Optional[str] = Field(None, q='id__startswith')
+    name: Optional[str] = Field(None, q='name__icontains')
+    binprovider: Optional[str] = Field(None, q='binprovider')
+    status: Optional[str] = Field(None, q='status')
+    machine_id: Optional[str] = Field(None, q='machine_id__startswith')
+    version: Optional[str] = Field(None, q='version__icontains')
+
+
+# ============================================================================
+# Machine Endpoints
+# ============================================================================
+
+@router.get("/machines", response=List[MachineSchema], url_name="get_machines")
+@paginate(CustomPagination)
+def get_machines(request, filters: MachineFilterSchema = Query(...)):
+    """List all machines."""
+    from archivebox.machine.models import Machine
+    return filters.filter(Machine.objects.all()).distinct()
+
+
+@router.get("/machine/{machine_id}", response=MachineSchema, url_name="get_machine")
+def get_machine(request, machine_id: str):
+    """Get a specific machine by ID."""
+    from archivebox.machine.models import Machine
+    from django.db.models import Q
+    return Machine.objects.get(Q(id__startswith=machine_id) | Q(hostname__iexact=machine_id))
+
+
+@router.get("/machine/current", response=MachineSchema, url_name="get_current_machine")
+def get_current_machine(request):
+    """Get the current machine."""
+    from archivebox.machine.models import Machine
+    return Machine.current()
+
+
+# ============================================================================
+
+
+# ============================================================================
+# Binary Endpoints
+# ============================================================================
+
+@router.get("/binaries", response=List[BinarySchema], url_name="get_binaries")
+@paginate(CustomPagination)
+def get_binaries(request, filters: BinaryFilterSchema = Query(...)):
+    """List all binaries."""
+    from archivebox.machine.models import Binary
+    return filters.filter(Binary.objects.all().select_related('machine', 'dependency')).distinct()
+
+
+@router.get("/binary/{binary_id}", response=BinarySchema, url_name="get_binary")
+def get_binary(request, binary_id: str):
+    """Get a specific binary by ID."""
+    from archivebox.machine.models import Binary
+    return Binary.objects.select_related('machine', 'dependency').get(id__startswith=binary_id)
+
+
+@router.get("/binary/by-name/{name}", response=List[BinarySchema], url_name="get_binaries_by_name")
+def get_binaries_by_name(request, name: str):
+    """Get all binaries with the given name."""
+    from archivebox.machine.models import Binary
+    return list(Binary.objects.filter(name__iexact=name).select_related('machine', 'dependency'))

archivebox/api/v1_workers.py

@@ -0,0 +1,160 @@
+__package__ = 'archivebox.api'
+
+from uuid import UUID
+from typing import List, Any
+from datetime import datetime
+
+from ninja import Router, Schema
+
+
+router = Router(tags=['Workers and Tasks'])
+
+
+class QueueItemSchema(Schema):
+    """Schema for a single item in a worker's queue."""
+    TYPE: str
+    id: UUID
+    status: str
+    retry_at: datetime | None
+    created_at: datetime
+    modified_at: datetime
+    description: str
+
+    @staticmethod
+    def resolve_TYPE(obj) -> str:
+        return f'{obj._meta.app_label}.{obj._meta.model_name}'
+
+    @staticmethod
+    def resolve_description(obj) -> str:
+        return str(obj)
+
+
+class WorkerSchema(Schema):
+    """Schema for a Worker type."""
+    name: str
+    model: str
+    max_tick_time: int
+    max_concurrent_tasks: int
+    poll_interval: float
+    idle_timeout: int
+    running_count: int
+    running_workers: List[dict[str, Any]]
+    queue_count: int
+    queue: List[QueueItemSchema]
+
+    @staticmethod
+    def resolve_model(obj) -> str:
+        Model = obj.get_model()
+        return f'{Model._meta.app_label}.{Model._meta.model_name}'
+
+    @staticmethod
+    def resolve_max_tick_time(obj) -> int:
+        return obj.MAX_TICK_TIME
+
+    @staticmethod
+    def resolve_max_concurrent_tasks(obj) -> int:
+        return obj.MAX_CONCURRENT_TASKS
+
+    @staticmethod
+    def resolve_poll_interval(obj) -> float:
+        return obj.POLL_INTERVAL
+
+    @staticmethod
+    def resolve_idle_timeout(obj) -> int:
+        return obj.IDLE_TIMEOUT
+
+    @staticmethod
+    def resolve_running_count(obj) -> int:
+        return len(obj.get_running_workers())
+
+    @staticmethod
+    def resolve_running_workers(obj) -> List[dict[str, Any]]:
+        return obj.get_running_workers()
+
+    @staticmethod
+    def resolve_queue_count(obj) -> int:
+        return obj.get_queue().count()
+
+    @staticmethod
+    def resolve_queue(obj) -> List[QueueItemSchema]:
+        return list(obj.get_queue()[:50])  # Limit to 50 items
+
+
+class OrchestratorSchema(Schema):
+    """Schema for the Orchestrator."""
+    is_running: bool
+    poll_interval: float
+    idle_timeout: int
+    max_workers_per_type: int
+    max_total_workers: int
+    total_worker_count: int
+    workers: List[WorkerSchema]
+
+
+@router.get("/orchestrator", response=OrchestratorSchema, url_name="get_orchestrator")
+def get_orchestrator(request):
+    """Get the orchestrator status and all worker queues."""
+    from archivebox.workers.orchestrator import Orchestrator
+    from archivebox.workers.worker import CrawlWorker, SnapshotWorker, ArchiveResultWorker
+
+    orchestrator = Orchestrator()
+
+    # Create temporary worker instances to query their queues
+    workers = [
+        CrawlWorker(worker_id=-1),
+        SnapshotWorker(worker_id=-1),
+        ArchiveResultWorker(worker_id=-1),
+    ]
+
+    return {
+        'is_running': orchestrator.is_running(),
+        'poll_interval': orchestrator.POLL_INTERVAL,
+        'idle_timeout': orchestrator.IDLE_TIMEOUT,
+        'max_workers_per_type': orchestrator.MAX_WORKERS_PER_TYPE,
+        'max_total_workers': orchestrator.MAX_TOTAL_WORKERS,
+        'total_worker_count': orchestrator.get_total_worker_count(),
+        'workers': workers,
+    }
+
+
+@router.get("/workers", response=List[WorkerSchema], url_name="get_workers")
+def get_workers(request):
+    """List all worker types and their current status."""
+    from archivebox.workers.worker import CrawlWorker, SnapshotWorker, ArchiveResultWorker
+
+    # Create temporary instances to query their queues
+    return [
+        CrawlWorker(worker_id=-1),
+        SnapshotWorker(worker_id=-1),
+        ArchiveResultWorker(worker_id=-1),
+    ]
+
+
+@router.get("/worker/{worker_name}", response=WorkerSchema, url_name="get_worker")
+def get_worker(request, worker_name: str):
+    """Get status and queue for a specific worker type."""
+    from archivebox.workers.worker import WORKER_TYPES
+
+    if worker_name not in WORKER_TYPES:
+        from ninja.errors import HttpError
+        raise HttpError(404, f"Unknown worker type: {worker_name}. Valid types: {list(WORKER_TYPES.keys())}")
+
+    WorkerClass = WORKER_TYPES[worker_name]
+    return WorkerClass(worker_id=-1)
+
+
+@router.get("/worker/{worker_name}/queue", response=List[QueueItemSchema], url_name="get_worker_queue")
+def get_worker_queue(request, worker_name: str, limit: int = 100):
+    """Get the current queue for a specific worker type."""
+    from archivebox.workers.worker import WORKER_TYPES
+
+    if worker_name not in WORKER_TYPES:
+        from ninja.errors import HttpError
+        raise HttpError(404, f"Unknown worker type: {worker_name}. Valid types: {list(WORKER_TYPES.keys())}")
+
+    WorkerClass = WORKER_TYPES[worker_name]
+    worker = WorkerClass(worker_id=-1)
+    return list(worker.get_queue()[:limit])
+
+
+# Progress endpoint moved to core.views.live_progress_view for simplicity

archivebox/base_models/__init__.py

@@ -0,0 +1 @@
+__package__ = 'archivebox.base_models'

archivebox/base_models/admin.py

@@ -0,0 +1,235 @@
+"""Base admin classes for models using UUIDv7."""
+
+__package__ = 'archivebox.base_models'
+
+import json
+
+from django import forms
+from django.contrib import admin
+from django.utils.html import format_html, mark_safe
+from django_object_actions import DjangoObjectActions
+
+
+class KeyValueWidget(forms.Widget):
+    """
+    A widget that renders JSON dict as editable key-value input fields
+    with + and - buttons to add/remove rows.
+    Includes autocomplete for available config keys from the plugin system.
+    """
+    template_name = None  # We render manually
+
+    class Media:
+        css = {
+            'all': []
+        }
+        js = []
+
+    def _get_config_options(self):
+        """Get available config options from plugins."""
+        try:
+            from archivebox.hooks import discover_plugin_configs
+            plugin_configs = discover_plugin_configs()
+            options = {}
+            for plugin_name, schema in plugin_configs.items():
+                for key, prop in schema.get('properties', {}).items():
+                    options[key] = {
+                        'plugin': plugin_name,
+                        'type': prop.get('type', 'string'),
+                        'default': prop.get('default', ''),
+                        'description': prop.get('description', ''),
+                    }
+            return options
+        except Exception:
+            return {}
+
+    def render(self, name, value, attrs=None, renderer=None):
+        # Parse JSON value to dict
+        if value is None:
+            data = {}
+        elif isinstance(value, str):
+            try:
+                data = json.loads(value) if value else {}
+            except json.JSONDecodeError:
+                data = {}
+        elif isinstance(value, dict):
+            data = value
+        else:
+            data = {}
+
+        widget_id = attrs.get('id', name) if attrs else name
+        config_options = self._get_config_options()
+
+        # Build datalist options
+        datalist_options = '\n'.join(
+            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
+            for key, opt in sorted(config_options.items())
+        )
+
+        # Build config metadata as JSON for JS
+        config_meta_json = json.dumps(config_options)
+
+        html = f'''
+        <div id="{widget_id}_container" class="key-value-editor" style="max-width: 700px;">
+            <datalist id="{widget_id}_keys">
+                {datalist_options}
+            </datalist>
+            <div id="{widget_id}_rows" class="key-value-rows">
+        '''
+
+        # Render existing key-value pairs
+        row_idx = 0
+        for key, val in data.items():
+            val_str = json.dumps(val) if not isinstance(val, str) else val
+            html += self._render_row(widget_id, row_idx, key, val_str)
+            row_idx += 1
+
+        # Always add one empty row for new entries
+        html += self._render_row(widget_id, row_idx, '', '')
+
+        html += f'''
+            </div>
+            <div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
+                <button type="button" onclick="addKeyValueRow_{widget_id}()"
+                        style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
+                    + Add Row
+                </button>
+                <span id="{widget_id}_hint" style="font-size: 11px; color: #666; font-style: italic;"></span>
+            </div>
+            <input type="hidden" name="{name}" id="{widget_id}" value="">
+            <script>
+                (function() {{
+                    var configMeta_{widget_id} = {config_meta_json};
+
+                    function showKeyHint_{widget_id}(key) {{
+                        var hint = document.getElementById('{widget_id}_hint');
+                        var meta = configMeta_{widget_id}[key];
+                        if (meta) {{
+                            hint.innerHTML = '<b>' + key + '</b>: ' + (meta.description || meta.type) +
+                                (meta.default !== '' ? ' <span style="color:#888">(default: ' + meta.default + ')</span>' : '');
+                        }} else {{
+                            hint.textContent = key ? 'Custom key: ' + key : '';
+                        }}
+                    }}
+
+                    function updateHiddenField_{widget_id}() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        var rows = container.querySelectorAll('.key-value-row');
+                        var result = {{}};
+                        rows.forEach(function(row) {{
+                            var keyInput = row.querySelector('.kv-key');
+                            var valInput = row.querySelector('.kv-value');
+                            if (keyInput && valInput && keyInput.value.trim()) {{
+                                var key = keyInput.value.trim();
+                                var val = valInput.value.trim();
+                                // Try to parse as JSON (for booleans, numbers, etc)
+                                try {{
+                                    if (val === 'true') result[key] = true;
+                                    else if (val === 'false') result[key] = false;
+                                    else if (val === 'null') result[key] = null;
+                                    else if (!isNaN(val) && val !== '') result[key] = Number(val);
+                                    else if ((val.startsWith('{{') && val.endsWith('}}')) ||
+                                             (val.startsWith('[') && val.endsWith(']')) ||
+                                             (val.startsWith('"') && val.endsWith('"')))
+                                        result[key] = JSON.parse(val);
+                                    else result[key] = val;
+                                }} catch(e) {{
+                                    result[key] = val;
+                                }}
+                            }}
+                        }});
+                        document.getElementById('{widget_id}').value = JSON.stringify(result);
+                    }}
+
+                    window.addKeyValueRow_{widget_id} = function() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        var rows = container.querySelectorAll('.key-value-row');
+                        var newIdx = rows.length;
+                        var newRow = document.createElement('div');
+                        newRow.className = 'key-value-row';
+                        newRow.style.cssText = 'display: flex; gap: 8px; margin-bottom: 6px; align-items: center;';
+                        newRow.innerHTML = '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
+                            'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;" ' +
+                            'onchange="updateHiddenField_{widget_id}()" oninput="updateHiddenField_{widget_id}(); showKeyHint_{widget_id}(this.value)" onfocus="showKeyHint_{widget_id}(this.value)">' +
+                            '<input type="text" class="kv-value" placeholder="value" ' +
+                            'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;" ' +
+                            'onchange="updateHiddenField_{widget_id}()" oninput="updateHiddenField_{widget_id}()">' +
+                            '<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
+                            'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>';
+                        container.appendChild(newRow);
+                        newRow.querySelector('.kv-key').focus();
+                    }};
+
+                    window.removeKeyValueRow_{widget_id} = function(btn) {{
+                        var row = btn.parentElement;
+                        row.remove();
+                        updateHiddenField_{widget_id}();
+                    }};
+
+                    window.showKeyHint_{widget_id} = showKeyHint_{widget_id};
+                    window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
+
+                    // Initialize on load
+                    document.addEventListener('DOMContentLoaded', function() {{
+                        updateHiddenField_{widget_id}();
+                    }});
+                    // Also run immediately in case DOM is already ready
+                    if (document.readyState !== 'loading') {{
+                        updateHiddenField_{widget_id}();
+                    }}
+
+                    // Update on any input change
+                    document.getElementById('{widget_id}_rows').addEventListener('input', updateHiddenField_{widget_id});
+                }})();
+            </script>
+        </div>
+        '''
+        return mark_safe(html)
+
+    def _render_row(self, widget_id, idx, key, value):
+        return f'''
+            <div class="key-value-row" style="display: flex; gap: 8px; margin-bottom: 6px; align-items: center;">
+                <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
+                       style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;"
+                       onchange="updateHiddenField_{widget_id}()" oninput="updateHiddenField_{widget_id}(); showKeyHint_{widget_id}(this.value)" onfocus="showKeyHint_{widget_id}(this.value)">
+                <input type="text" class="kv-value" value="{self._escape(value)}" placeholder="value"
+                       style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;"
+                       onchange="updateHiddenField_{widget_id}()" oninput="updateHiddenField_{widget_id}()">
+                <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
+                        style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>
+            </div>
+        '''
+
+    def _escape(self, s):
+        """Escape HTML special chars in attribute values."""
+        if not s:
+            return ''
+        return str(s).replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
+
+    def value_from_datadict(self, data, files, name):
+        value = data.get(name, '{}')
+        return value
+
+
+class ConfigEditorMixin:
+    """
+    Mixin for admin classes with a config JSON field.
+
+    Provides a key-value editor widget with autocomplete for available config keys.
+    """
+
+    def formfield_for_dbfield(self, db_field, request, **kwargs):
+        """Use KeyValueWidget for the config JSON field."""
+        if db_field.name == 'config':
+            kwargs['widget'] = KeyValueWidget()
+        return super().formfield_for_dbfield(db_field, request, **kwargs)
+
+
+class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
+    list_display = ('id', 'created_at', 'created_by')
+    readonly_fields = ('id', 'created_at', 'modified_at')
+
+    def get_form(self, request, obj=None, **kwargs):
+        form = super().get_form(request, obj, **kwargs)
+        if 'created_by' in form.base_fields:
+            form.base_fields['created_by'].initial = request.user
+        return form

archivebox/base_models/apps.py

@@ -0,0 +1,7 @@
+# from django.apps import AppConfig
+
+
+# class BaseModelsConfig(AppConfig):
+#     default_auto_field = 'django.db.models.BigAutoField'
+
+#     name = 'base_models'

archivebox/base_models/models.py

@@ -0,0 +1,154 @@
+"""Base models using UUIDv7 for all id fields."""
+
+__package__ = 'archivebox.base_models'
+
+import io
+import csv
+import json
+from uuid import UUID
+from archivebox.uuid_compat import uuid7
+from typing import Any, Iterable, ClassVar
+from pathlib import Path
+
+from django.contrib import admin
+from django.db import models
+from django.db.models import F
+from django.utils import timezone
+from django.contrib.auth import get_user_model
+from django.urls import reverse_lazy
+from django.conf import settings
+
+from django_stubs_ext.db.models import TypedModelMeta
+
+from archivebox import DATA_DIR
+from archivebox.misc.util import to_json
+from archivebox.misc.hashing import get_dir_info
+
+
+def get_or_create_system_user_pk(username='system'):
+    User = get_user_model()
+    # If there's exactly one superuser, use that for all system operations
+    if User.objects.filter(is_superuser=True).count() == 1:
+        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
+    # Otherwise get or create the system user
+    user, _ = User.objects.get_or_create(
+        username=username,
+        defaults={'is_staff': True, 'is_superuser': True, 'email': '', 'password': '!'}
+    )
+    return user.pk
+
+
+class AutoDateTimeField(models.DateTimeField):
+    """DateTimeField that automatically updates on save (legacy compatibility)."""
+    def pre_save(self, model_instance, add):
+        if add or not getattr(model_instance, self.attname):
+            value = timezone.now()
+            setattr(model_instance, self.attname, value)
+            return value
+        return super().pre_save(model_instance, add)
+
+
+class ModelWithUUID(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False, db_index=True)
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    def __str__(self):
+        return f'[{self.id}] {self.__class__.__name__}'
+
+    @property
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
+
+    @property
+    def api_url(self) -> str:
+        return reverse_lazy('api-1:get_any', args=[self.id])
+
+    @property
+    def api_docs_url(self) -> str:
+        return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}'
+
+    def as_json(self, keys: Iterable[str] = ()) -> dict:
+        default_keys = ('id', 'created_at', 'modified_at')
+        return {key: getattr(self, key) for key in (keys or default_keys) if hasattr(self, key)}
+
+
+class ModelWithSerializers(ModelWithUUID):
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    def as_csv_row(self, keys: Iterable[str] = (), separator: str = ',') -> str:
+        buffer = io.StringIO()
+        csv.writer(buffer, delimiter=separator).writerow(str(getattr(self, key, '')) for key in (keys or self.as_json().keys()))
+        return buffer.getvalue()
+
+    def as_jsonl_row(self, keys: Iterable[str] = (), **json_kwargs) -> str:
+        return json.dumps({key: getattr(self, key, '') for key in (keys or self.as_json().keys())}, sort_keys=True, indent=None, **json_kwargs)
+
+
+class ModelWithNotes(models.Model):
+    """Mixin for models with a notes field."""
+    notes = models.TextField(blank=True, null=False, default='')
+
+    class Meta:
+        abstract = True
+
+
+class ModelWithHealthStats(models.Model):
+    """Mixin for models with health tracking fields."""
+    num_uses_failed = models.PositiveIntegerField(default=0)
+    num_uses_succeeded = models.PositiveIntegerField(default=0)
+
+    class Meta:
+        abstract = True
+
+    @property
+    def health(self) -> int:
+        total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
+        return round((self.num_uses_succeeded / total) * 100)
+
+    def increment_health_stats(self, success: bool):
+        """Atomically increment success or failure counter using F() expression."""
+        field = 'num_uses_succeeded' if success else 'num_uses_failed'
+        type(self).objects.filter(pk=self.pk).update(**{field: F(field) + 1})
+
+
+class ModelWithConfig(models.Model):
+    """Mixin for models with a JSON config field."""
+    config = models.JSONField(default=dict, null=True, blank=True, editable=True)
+
+    class Meta:
+        abstract = True
+
+
+class ModelWithOutputDir(ModelWithSerializers):
+    class Meta:
+        abstract = True
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        self.OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+        self.save_json_index()
+
+    @property
+    def output_dir_parent(self) -> str:
+        return f'{self._meta.model_name}s'
+
+    @property
+    def output_dir_name(self) -> str:
+        return str(self.id)
+
+    @property
+    def output_dir_str(self) -> str:
+        return f'{self.output_dir_parent}/{self.output_dir_name}'
+
+    @property
+    def OUTPUT_DIR(self) -> Path:
+        return DATA_DIR / self.output_dir_str
+
+    def save_json_index(self):
+        (self.OUTPUT_DIR / 'index.json').write_text(to_json(self.as_json()))

archivebox/builtin_plugins/base/apps.py

@@ -1,82 +0,0 @@
-import sys
-import inspect
-from typing import List, Dict, Any, Optional
-from pathlib import Path
-
-import django
-from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
-from django.db.backends.sqlite3.base import Database as sqlite3
-
-from pydantic import (
-    Field,
-    SerializeAsAny,
-)
-
-from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
-
-from plugantic.extractors import Extractor, ExtractorName
-from plugantic.plugins import Plugin
-from plugantic.configs import ConfigSet, ConfigSectionName
-from plugantic.replayers import Replayer
-
-
-class PythonBinary(Binary):
-    name: BinName = 'python'
-
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides: Dict[str, Any] = {
-        'env': {
-            'subdeps': \
-                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
-            'abspath': \
-                lambda: sys.executable,
-            'version': \
-                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
-        },
-    }
-
-class SqliteBinary(Binary):
-    name: BinName = 'sqlite'
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'env': {
-            'abspath': \
-                lambda: inspect.getfile(sqlite3),
-            'version': \
-                lambda: sqlite3.version,
-        },
-    }
-
-class DjangoBinary(Binary):
-    name: BinName = 'django'
-
-    providers_supported: List[BinProvider] = [EnvProvider()]
-    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
-        'env': {
-            'abspath': \
-                lambda: inspect.getfile(django),
-            'version': \
-                lambda: django.VERSION[:3],
-        },
-    }
-
-
-class BasicReplayer(Replayer):
-    name: str = 'basic'
-
-
-class BasePlugin(Plugin):
-    name: str = 'base'
-    configs: List[SerializeAsAny[ConfigSet]] = []
-    binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = []
-    replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()]
-
-
-PLUGINS = [BasePlugin()]
-
-
-class BaseConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'builtin_plugins.base'

archivebox/builtin_plugins/base/models.py

@@ -1,3 +0,0 @@
-from django.db import models
-
-# Create your models here.

archivebox/builtin_plugins/base/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/builtin_plugins/singlefile/apps.py

@@ -1,94 +0,0 @@
-from typing import List, Optional
-from pathlib import Path
-
-from django.apps import AppConfig
-from django.core.checks import Tags, Warning, register
-
-from pydantic import (
-    Field,
-    SerializeAsAny,
-)
-
-from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider
-
-from plugantic.extractors import Extractor, ExtractorName
-from plugantic.plugins import Plugin
-from plugantic.configs import ConfigSet, ConfigSectionName
-
-
-###################### Config ##########################
-
-class SinglefileToggleConfig(ConfigSet):
-    section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
-
-    SAVE_SINGLEFILE: bool = True
-
-
-class SinglefileDependencyConfig(ConfigSet):
-    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
-
-    SINGLEFILE_BINARY: str = Field(default='wget')
-    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
-    SINGLEFILE_EXTRA_ARGS: List[str] = []
-    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
-
-class SinglefileOptionsConfig(ConfigSet):
-    section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
-
-    # loaded from shared config
-    SINGLEFILE_USER_AGENT: str = Field(default='', alias='USER_AGENT')
-    SINGLEFILE_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
-    SINGLEFILE_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
-    SINGLEFILE_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
-    SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
-
-
-
-DEFAULT_CONFIG = {
-    'CHECK_SSL_VALIDITY': False,
-    'SAVE_SINGLEFILE': True,
-    'TIMEOUT': 120,
-}
-
-PLUGIN_CONFIG = [
-    SinglefileToggleConfig(**DEFAULT_CONFIG),
-    SinglefileDependencyConfig(**DEFAULT_CONFIG),
-    SinglefileOptionsConfig(**DEFAULT_CONFIG),
-]
-
-###################### Binaries ############################
-
-class SinglefileBinary(Binary):
-    name: BinName = 'single-file'
-    providers_supported: List[BinProvider] = [EnvProvider(), NpmProvider()]
-
-
-###################### Extractors ##########################
-
-class SinglefileExtractor(Extractor):
-    name: ExtractorName = 'singlefile'
-    binary: Binary = SinglefileBinary()
-
-    def get_output_path(self, snapshot) -> Path:
-        return Path(snapshot.link_dir) / 'singlefile.html'
-
-
-###################### Plugins #############################
-
-
-class SinglefilePlugin(Plugin):
-    name: str = 'singlefile'
-    configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG]
-    binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()]
-    extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()]
-
-PLUGINS = [SinglefilePlugin()]
-
-###################### Django Apps #########################
-
-class SinglefileConfig(AppConfig):
-    name = 'builtin_plugins.singlefile'
-    verbose_name = 'SingleFile'
-
-    def ready(self):
-        print('Loaded singlefile plugin')

archivebox/builtin_plugins/singlefile/config.yaml

@@ -1,66 +0,0 @@
-name: singlefile
-plugin_version: '0.0.1'
-plugin_spec: '0.0.1'
-
-binaries:
-    singlefile:
-        providers:
-            - env
-            - npm
-
-commands:
-    - singlefile.exec
-    - singlefile.extract
-    - singlefile.should_extract
-    - singlefile.get_output_path
-
-extractors:
-    singlefile:
-        binary: singlefile
-        test: singlefile.should_extract
-        extract: singlefile.extract
-        output_files:
-            - singlefile.html
-
-configs:
-    ARCHIVE_METHOD_TOGGLES:
-        SAVE_SINGLEFILE:
-            type: bool
-            default: true
-
-    DEPENDENCY_CONFIG:
-        SINGLEFILE_BINARY:
-            type: str
-            default: wget
-        SINGLEFILE_ARGS:
-            type: Optional[List[str]]
-            default: null
-        SINGLEFILE_EXTRA_ARGS:
-            type: List[str]
-            default: []
-        SINGLEFILE_DEFAULT_ARGS:
-            type: List[str]
-            default: 
-            - "--timeout={TIMEOUT-10}"
-
-    ARCHIVE_METHOD_OPTIONS:
-        SINGLEFILE_USER_AGENT:
-            type: str
-            default: ""
-            alias: USER_AGENT
-        SINGLEFILE_TIMEOUT:
-            type: int
-            default: 60
-            alias: TIMEOUT
-        SINGLEFILE_CHECK_SSL_VALIDITY:
-            type: bool
-            default: true
-            alias: CHECK_SSL_VALIDITY
-        SINGLEFILE_RESTRICT_FILE_NAMES:
-            type: str
-            default: windows
-            alias: RESTRICT_FILE_NAMES
-        SINGLEFILE_COOKIES_FILE:
-            type: Optional[Path]
-            default: null
-            alias: COOKIES_FILE

archivebox/builtin_plugins/singlefile/tests.py

@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.

archivebox/cli/__init__.py

@@ -1,196 +1,153 @@
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox'
-
 import os
 import sys
-import argparse
-import threading
-from time import sleep
-
-from typing import Optional, Dict, List, IO, Union, Iterable
-from pathlib import Path
-
-from ..config import OUTPUT_DIR, check_data_folder, check_migrations, stderr
-
 from importlib import import_module
 
-BUILTIN_LIST = list
+import rich_click as click
+from rich import print
 
-CLI_DIR = Path(__file__).resolve().parent
-
-# these common commands will appear sorted before any others for ease-of-use
-meta_cmds = ('help', 'version')                               # dont require valid data folder at all
-main_cmds = ('init', 'config', 'setup')                       # dont require existing db present
-archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
-fake_db = ("oneshot",)                                        # use fake in-memory db
-
-display_first = (*meta_cmds, *main_cmds, *archive_cmds)
-
-# every imported command module must have these properties in order to be valid
-required_attrs = ('__package__', '__command__', 'main')
-
-# basic checks to make sure imported files are valid subcommands
-is_cli_module = lambda fname: fname.startswith('archivebox_') and fname.endswith('.py')
-is_valid_cli_module = lambda module, subcommand: (
-    all(hasattr(module, attr) for attr in required_attrs)
-    and module.__command__.split(' ')[-1] == subcommand
-)
+from archivebox.config.version import VERSION
 
 
-IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread')  # threads we dont have to wait for before exiting
+
+if '--debug' in sys.argv:
+    os.environ['DEBUG'] = 'True'
+    sys.argv.remove('--debug')
 
 
-def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: Iterable[str]=IGNORED_BG_THREADS, timeout: int=60) -> int:
-    """
-    Block until the specified threads exit. e.g. pass thread_names=('default_hook_handler',) to wait for webhooks.
-    Useful for waiting for signal handlers, webhooks, etc. to finish running after a mgmt command completes.
-    """
+class ArchiveBoxGroup(click.Group):
+    """lazy loading click group for archivebox commands"""
+    meta_commands = {
+        'help': 'archivebox.cli.archivebox_help.main',
+        'version': 'archivebox.cli.archivebox_version.main',
+        'mcp': 'archivebox.cli.archivebox_mcp.main',
+    }
+    setup_commands = {
+        'init': 'archivebox.cli.archivebox_init.main',
+        'install': 'archivebox.cli.archivebox_install.main',
+    }
+    # Model commands (CRUD operations via subcommands)
+    model_commands = {
+        'crawl': 'archivebox.cli.archivebox_crawl.main',
+        'snapshot': 'archivebox.cli.archivebox_snapshot.main',
+        'archiveresult': 'archivebox.cli.archivebox_archiveresult.main',
+        'tag': 'archivebox.cli.archivebox_tag.main',
+        'binary': 'archivebox.cli.archivebox_binary.main',
+        'process': 'archivebox.cli.archivebox_process.main',
+        'machine': 'archivebox.cli.archivebox_machine.main',
+    }
+    archive_commands = {
+        # High-level commands
+        'add': 'archivebox.cli.archivebox_add.main',
+        'run': 'archivebox.cli.archivebox_run.main',
+        'update': 'archivebox.cli.archivebox_update.main',
+        'status': 'archivebox.cli.archivebox_status.main',
+        'config': 'archivebox.cli.archivebox_config.main',
+        'schedule': 'archivebox.cli.archivebox_schedule.main',
+        'server': 'archivebox.cli.archivebox_server.main',
+        'shell': 'archivebox.cli.archivebox_shell.main',
+        'manage': 'archivebox.cli.archivebox_manage.main',
+        # Introspection commands
+        'pluginmap': 'archivebox.cli.archivebox_pluginmap.main',
+        # Worker command
+        'worker': 'archivebox.cli.archivebox_worker.main',
+    }
+    all_subcommands = {
+        **meta_commands,
+        **setup_commands,
+        **model_commands,
+        **archive_commands,
+    }
+    renamed_commands = {
+        'setup': 'install',
+        'import': 'add',
+        'archive': 'add',
+        # Old commands replaced by new model commands
+        'orchestrator': 'run',
+        'extract': 'archiveresult',
+    }
+    
+    @classmethod
+    def get_canonical_name(cls, cmd_name):
+        return cls.renamed_commands.get(cmd_name, cmd_name)
+    
 
-    wait_for_all: bool = thread_names == ()
+    def get_command(self, ctx, cmd_name):
+        # handle renamed commands
+        if cmd_name in self.renamed_commands:
+            new_name = self.renamed_commands[cmd_name]
+            print(f' [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`')
+            cmd_name = new_name
+            ctx.invoked_subcommand = cmd_name
+        
+        # handle lazy loading of commands
+        if cmd_name in self.all_subcommands:
+            return self._lazy_load(cmd_name)
+        
+        # fall-back to using click's default command lookup
+        return super().get_command(ctx, cmd_name)
 
-    thread_matches = lambda thread, ptns: any(ptn in repr(thread) for ptn in ptns)
-
-    should_wait = lambda thread: (
-        not thread_matches(thread, ignore_names)
-        and (wait_for_all or thread_matches(thread, thread_names)))
-
-    for tries in range(timeout):
-        all_threads = [*threading.enumerate()]
-        blocking_threads = [*filter(should_wait, all_threads)]
-        threads_summary = ', '.join(repr(t) for t in blocking_threads)
-        if blocking_threads:
-            sleep(1)
-            if tries == 5:                            # only show stderr message if we need to wait more than 5s
-                stderr(
-                    f'[…] Waiting up to {timeout}s for background jobs (e.g. webhooks) to finish...',
-                    threads_summary,
-                )
-        else:
-            return tries
-
-    raise Exception('Background threads failed to exit after {tries}s: {threads_summary}')
+    @classmethod
+    def _lazy_load(cls, cmd_name):
+        import_path = cls.all_subcommands[cmd_name]
+        modname, funcname = import_path.rsplit('.', 1)
+        
+        # print(f'LAZY LOADING {import_path}')
+        mod = import_module(modname)
+        func = getattr(mod, funcname)
+        
+        if not hasattr(func, '__doc__'):
+            raise ValueError(f'lazy loading of {import_path} failed - no docstring found on method')
+        
+        # if not isinstance(cmd, click.BaseCommand):
+            # raise ValueError(f'lazy loading of {import_path} failed - not a click command')
+            
+        return func
 
 
-def list_subcommands() -> Dict[str, str]:
-    """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
+@click.group(cls=ArchiveBoxGroup, invoke_without_command=True)
+@click.option('--help', '-h', is_flag=True, help='Show help')
+@click.version_option(VERSION, '-v', '--version', package_name='archivebox', message='%(version)s')
+@click.pass_context
+def cli(ctx, help=False):
+    """ArchiveBox: The self-hosted internet archive"""
+    
+    subcommand = ArchiveBoxGroup.get_canonical_name(ctx.invoked_subcommand)
+    
+    # if --help is passed or no subcommand is given, show custom help message
+    if help or ctx.invoked_subcommand is None:
+        ctx.invoke(ctx.command.get_command(ctx, 'help'))
+    
+    # if the subcommand is in archive_commands or model_commands,
+    # then we need to set up the django environment and check that we're in a valid data folder
+    if subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands:
+        # print('SETUP DJANGO AND CHECK DATA FOLDER')
+        try:
+            from archivebox.config.django import setup_django
+            from archivebox.misc.checks import check_data_folder
+            setup_django()
+            check_data_folder()
+        except Exception as e:
+            print(f'[red][X] Error setting up Django or checking data folder: {e}[/red]', file=sys.stderr)
+            if subcommand not in ('manage', 'shell'):   # not all management commands need django to be setup beforehand
+                raise
+            
 
-    COMMANDS = []
-    for filename in os.listdir(CLI_DIR):
-        if is_cli_module(filename):
-            subcommand = filename.replace('archivebox_', '').replace('.py', '')
-            module = import_module('.archivebox_{}'.format(subcommand), __package__)
-            assert is_valid_cli_module(module, subcommand)
-            COMMANDS.append((subcommand, module.main.__doc__))
-            globals()[subcommand] = module.main
+def main(args=None, prog_name=None, stdin=None):
+    # show `docker run archivebox xyz` in help messages if running in docker
+    IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE', 'yes')
+    IS_TTY = sys.stdin.isatty()
+    prog_name = prog_name or (f'docker compose run{"" if IS_TTY else " -T"} archivebox' if IN_DOCKER else 'archivebox')
+    
+    # stdin param allows passing input data from caller (used by __main__.py)
+    # currently not used by click-based CLI, but kept for backwards compatibility
 
-    display_order = lambda cmd: (
-        display_first.index(cmd[0])
-        if cmd[0] in display_first else
-        100 + len(cmd[0])
-    )
-
-    return dict(sorted(COMMANDS, key=display_order))
-
-
-def run_subcommand(subcommand: str,
-                   subcommand_args: List[str]=None,
-                   stdin: Optional[IO]=None,
-                   pwd: Union[Path, str, None]=None) -> None:
-    """Run a given ArchiveBox subcommand with the given list of args"""
-
-    subcommand_args = subcommand_args or []
-
-    if subcommand not in meta_cmds:
-        from ..config import setup_django
-
-        cmd_requires_db = subcommand in archive_cmds
-        init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
-
-        if cmd_requires_db:
-            check_data_folder(pwd)
-
-        setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)
-
-        if cmd_requires_db:
-            check_migrations()
-
-    module = import_module('.archivebox_{}'.format(subcommand), __package__)
-    module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore
-
-    # wait for webhooks, signals, and other background jobs to finish before exit
-    wait_for_bg_threads_to_exit(timeout=60)
-
-
-SUBCOMMANDS = list_subcommands()
-
-class NotProvided:
-    pass
-
-
-def main(args: Optional[List[str]]=NotProvided, stdin: Optional[IO]=NotProvided, pwd: Optional[str]=None) -> None:
-    args = sys.argv[1:] if args is NotProvided else args
-    stdin = sys.stdin if stdin is NotProvided else stdin
-
-    subcommands = list_subcommands()
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description='ArchiveBox: The self-hosted internet archive',
-        add_help=False,
-    )
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        '--help', '-h',
-        action='store_true',
-        help=subcommands['help'],
-    )
-    group.add_argument(
-        '--version',
-        action='store_true',
-        help=subcommands['version'],
-    )
-    group.add_argument(
-        "subcommand",
-        type=str,
-        help= "The name of the subcommand to run",
-        nargs='?',
-        choices=subcommands.keys(),
-        default=None,
-    )
-    parser.add_argument(
-        "subcommand_args",
-        help="Arguments for the subcommand",
-        nargs=argparse.REMAINDER,
-    )
-    command = parser.parse_args(args or ())
-
-    if command.version:
-        command.subcommand = 'version'
-    elif command.help or command.subcommand is None:
-        command.subcommand = 'help'
-
-    if command.subcommand not in ('help', 'version', 'status'):
-        from ..logging_util import log_cli_command
-
-        log_cli_command(
-            subcommand=command.subcommand,
-            subcommand_args=command.subcommand_args,
-            stdin=stdin,
-            pwd=pwd or OUTPUT_DIR
-        )
-
-    run_subcommand(
-        subcommand=command.subcommand,
-        subcommand_args=command.subcommand_args,
-        stdin=stdin,
-        pwd=pwd or OUTPUT_DIR,
-    )
-
-
-__all__ = (
-    'SUBCOMMANDS',
-    'list_subcommands',
-    'run_subcommand',
-    *SUBCOMMANDS.keys(),
-)
+    try:
+        cli(args=args, prog_name=prog_name)
+    except KeyboardInterrupt:
+        print('\n\n[red][X] Got CTRL+C. Exiting...[/red]')
 
 
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_add.py

@@ -4,137 +4,157 @@ __package__ = 'archivebox.cli'
 __command__ = 'archivebox add'
 
 import sys
-import argparse
 
-from typing import List, Optional, IO
+from typing import TYPE_CHECKING
 
-from ..main import add
-from ..util import docstring
-from ..parsers import PARSERS
-from ..config import OUTPUT_DIR, ONLY_NEW
-from ..logging_util import SmartFormatter, accept_stdin, stderr
+import rich_click as click
+
+from django.utils import timezone
+from django.db.models import QuerySet
+
+from archivebox.misc.util import enforce_types, docstring
+from archivebox import CONSTANTS
+from archivebox.config.common import ARCHIVING_CONFIG
+from archivebox.config.permissions import USER, HOSTNAME
 
 
+if TYPE_CHECKING:
+    from archivebox.core.models import Snapshot
+
+
+@enforce_types
+def add(urls: str | list[str],
+        depth: int | str=0,
+        tag: str='',
+        parser: str="auto",
+        plugins: str="",
+        persona: str='Default',
+        overwrite: bool=False,
+        update: bool=not ARCHIVING_CONFIG.ONLY_NEW,
+        index_only: bool=False,
+        bg: bool=False,
+        created_by_id: int | None=None) -> QuerySet['Snapshot']:
+    """Add a new URL or list of URLs to your archive.
+
+    The flow is:
+    1. Save URLs to sources file
+    2. Create Crawl with URLs and max_depth
+    3. Orchestrator creates Snapshots from Crawl URLs (depth=0)
+    4. Orchestrator runs parser extractors on root snapshots
+    5. Parser extractors output to urls.jsonl
+    6. URLs are added to Crawl.urls and child Snapshots are created
+    7. Repeat until max_depth is reached
+    """
+
+    from rich import print
+
+    depth = int(depth)
+
+    assert depth in (0, 1, 2, 3, 4), 'Depth must be 0-4'
+
+    # import models once django is set up
+    from archivebox.core.models import Snapshot
+    from archivebox.crawls.models import Crawl
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.workers.orchestrator import Orchestrator
+
+    created_by_id = created_by_id or get_or_create_system_user_pk()
+
+    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
+    sources_file = CONSTANTS.SOURCES_DIR / f'{timezone.now().strftime("%Y-%m-%d__%H-%M-%S")}__cli_add.txt'
+    sources_file.parent.mkdir(parents=True, exist_ok=True)
+    sources_file.write_text(urls if isinstance(urls, str) else '\n'.join(urls))
+
+    # 2. Create a new Crawl with inline URLs
+    cli_args = [*sys.argv]
+    if cli_args[0].lower().endswith('archivebox'):
+        cli_args[0] = 'archivebox'
+    cmd_str = ' '.join(cli_args)
+
+    timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
+
+    # Read URLs directly into crawl
+    urls_content = sources_file.read_text()
+
+    crawl = Crawl.objects.create(
+        urls=urls_content,
+        max_depth=depth,
+        tags_str=tag,
+        label=f'{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]',
+        created_by_id=created_by_id,
+        config={
+            'ONLY_NEW': not update,
+            'INDEX_ONLY': index_only,
+            'OVERWRITE': overwrite,
+            'PLUGINS': plugins,
+            'DEFAULT_PERSONA': persona or 'Default',
+            'PARSER': parser,
+        }
+    )
+
+    print(f'[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]')
+    first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ''
+    print(f'    [dim]First URL: {first_url}[/dim]')
+
+    # 3. The CrawlMachine will create the root Snapshot when started
+    #    If URLs are from a file: first URL = file:///path/to/sources/...txt
+    #    Parser extractors will run on it and discover more URLs
+    #    Those URLs become child Snapshots (depth=1)
+
+    if index_only:
+        # Just create the crawl but don't start processing
+        print('[yellow]\\[*] Index-only mode - crawl created but not started[/yellow]')
+        # Create snapshots for all URLs in the crawl
+        for url in crawl.get_urls_list():
+            Snapshot.objects.update_or_create(
+                crawl=crawl, url=url,
+                defaults={
+                    'status': Snapshot.INITIAL_STATE,
+                    'retry_at': timezone.now(),
+                    'timestamp': str(timezone.now().timestamp()),
+                    'depth': 0,
+                },
+            )
+        return crawl.snapshot_set.all()
+
+    # 5. Start the orchestrator to process the queue
+    #    The orchestrator will:
+    #    - Process Crawl -> create root Snapshot
+    #    - Process root Snapshot -> run parser extractors -> discover URLs
+    #    - Create child Snapshots from discovered URLs
+    #    - Process child Snapshots -> run extractors
+    #    - Repeat until max_depth reached
+
+    if bg:
+        # Background mode: just queue work and return (orchestrator via server will pick it up)
+        print('[yellow]\\[*] URLs queued. Orchestrator will process them (run `archivebox server` if not already running).[/yellow]')
+    else:
+        # Foreground mode: run orchestrator inline until all work is done
+        print(f'[green]\\[*] Starting orchestrator to process crawl...[/green]')
+        orchestrator = Orchestrator(exit_on_idle=True)
+        orchestrator.runloop()  # Block until complete
+
+    # 6. Return the list of Snapshots in this crawl
+    return crawl.snapshot_set.all()
+
+
+@click.command()
+@click.option('--depth', '-d', type=click.Choice([str(i) for i in range(5)]), default='0', help='Recursively archive linked pages up to N hops away')
+@click.option('--tag', '-t', default='', help='Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3')
+@click.option('--parser', default='auto', help='Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)')
+@click.option('--plugins', '-p', default='', help='Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...')
+@click.option('--persona', default='Default', help='Authentication profile to use when archiving')
+@click.option('--overwrite', '-F', is_flag=True, help='Overwrite existing data if URLs have been archived previously')
+@click.option('--update', is_flag=True, default=ARCHIVING_CONFIG.ONLY_NEW, help='Retry any previously skipped/failed URLs when re-adding them')
+@click.option('--index-only', is_flag=True, help='Just add the URLs to the index without archiving them now')
+@click.option('--bg', is_flag=True, help='Run archiving in background (start orchestrator and return immediately)')
+@click.argument('urls', nargs=-1, type=click.Path())
 @docstring(add.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=add.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        '--tag', '-t',
-        type=str,
-        default='',
-        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
-    )
-    parser.add_argument(
-        '--update', #'-u',
-        action='store_true',
-        default=not ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
-        help="Also retry previously skipped/failed links when adding new links",
-    )
-    parser.add_argument(
-        '--update-all', #'-n',
-        action='store_true',
-        default=False, 
-        help="Also update ALL links in index when finished adding new links",
-    )
-    parser.add_argument(
-        '--index-only', #'-o',
-        action='store_true',
-        help="Add the links to the main index without archiving them",
-    )
-    parser.add_argument(
-        'urls',
-        nargs='*',
-        type=str,
-        default=None,
-        help=(
-            'URLs or paths to archive e.g.:\n'
-            '    https://getpocket.com/users/USERNAME/feed/all\n'
-            '    https://example.com/some/rss/feed.xml\n'
-            '    https://example.com\n'
-            '    ~/Downloads/firefox_bookmarks_export.html\n'
-            '    ~/Desktop/sites_list.csv\n'
-        )
-    )
-    parser.add_argument(
-        "--depth",
-        action="store",
-        default=0,
-        choices=[0, 1],
-        type=int,
-        help="Recursively archive all linked pages up to this many hops away"
-    )
-    parser.add_argument(
-        "--overwrite",
-        default=False,
-        action="store_true",
-        help="Re-archive URLs from scratch, overwriting any existing files"
-    )
-    parser.add_argument(
-        "--init", #'-i',
-        action='store_true',
-        help="Init/upgrade the curent data directory before adding",
-    )
-    parser.add_argument(
-        "--extract",
-        type=str,
-        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
-              This does not take precedence over the configuration",
-        default=""
-    )
-    parser.add_argument(
-        "--parser",
-        type=str,
-        help="Parser used to read inputted URLs.",
-        default="auto",
-        choices=["auto", *PARSERS.keys()],
-    )
-    command = parser.parse_args(args or ())
-    urls = command.urls
+def main(**kwargs):
+    """Add a new URL or list of URLs to your archive"""
 
-    stdin_urls = ''
-    if not urls:
-        stdin_urls = accept_stdin(stdin)
-
-    if (stdin_urls and urls) or (not stdin and not urls):
-        stderr(
-            '[X] You must pass URLs/paths to add via stdin or CLI arguments.\n',
-            color='red',
-        )
-        raise SystemExit(2)
-    add(
-        urls=stdin_urls or urls,
-        depth=command.depth,
-        tag=command.tag,
-        update=command.update,
-        update_all=command.update_all,
-        index_only=command.index_only,
-        overwrite=command.overwrite,
-        init=command.init,
-        extractors=command.extract,
-        parser=command.parser,
-        out_dir=pwd or OUTPUT_DIR,
-    )
+    add(**kwargs)
 
 
 if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
-
-
-# TODO: Implement these
-#
-# parser.add_argument(
-#     '--mirror', #'-m',
-#     action='store_true',
-#     help='Archive an entire site (finding all linked pages below it on the same domain)',
-# )
-# parser.add_argument(
-#     '--crawler', #'-r',
-#     choices=('depth_first', 'breadth_first'),
-#     help='Controls which crawler to use in order to find outlinks in a given page',
-#     default=None,
-# )
+    main()

archivebox/cli/archivebox_archiveresult.py

@@ -0,0 +1,380 @@
+#!/usr/bin/env python3
+
+"""
+archivebox archiveresult <action> [args...] [--filters]
+
+Manage ArchiveResult records (plugin extraction results).
+
+Actions:
+    create  - Create ArchiveResults for Snapshots (queue extractions)
+    list    - List ArchiveResults as JSONL (with optional filters)
+    update  - Update ArchiveResults from stdin JSONL
+    delete  - Delete ArchiveResults from stdin JSONL
+
+Examples:
+    # Create ArchiveResults for snapshots (queue for extraction)
+    archivebox snapshot list --status=queued | archivebox archiveresult create
+    archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
+
+    # List with filters
+    archivebox archiveresult list --status=failed
+    archivebox archiveresult list --plugin=screenshot --status=succeeded
+
+    # Update (reset failed extractions to queued)
+    archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued
+
+    # Delete
+    archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
+
+    # Re-run failed extractions
+    archivebox archiveresult list --status=failed | archivebox run
+"""
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox archiveresult'
+
+import sys
+from typing import Optional
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+# =============================================================================
+# CREATE
+# =============================================================================
+
+def create_archiveresults(
+    snapshot_id: Optional[str] = None,
+    plugin: Optional[str] = None,
+    status: str = 'queued',
+) -> int:
+    """
+    Create ArchiveResults for Snapshots.
+
+    Reads Snapshot records from stdin and creates ArchiveResult entries.
+    Pass-through: Non-Snapshot/ArchiveResult records are output unchanged.
+    If --plugin is specified, only creates results for that plugin.
+    Otherwise, creates results for all pending plugins.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from django.utils import timezone
+
+    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
+    from archivebox.core.models import Snapshot, ArchiveResult
+
+    is_tty = sys.stdout.isatty()
+
+    # If snapshot_id provided directly, use that
+    if snapshot_id:
+        try:
+            snapshots = [Snapshot.objects.get(id=snapshot_id)]
+            pass_through_records = []
+        except Snapshot.DoesNotExist:
+            rprint(f'[red]Snapshot not found: {snapshot_id}[/red]', file=sys.stderr)
+            return 1
+    else:
+        # Read from stdin
+        records = list(read_stdin())
+        if not records:
+            rprint('[yellow]No Snapshot records provided via stdin[/yellow]', file=sys.stderr)
+            return 1
+
+        # Separate snapshot records from pass-through records
+        snapshot_ids = []
+        pass_through_records = []
+
+        for record in records:
+            record_type = record.get('type', '')
+
+            if record_type == TYPE_SNAPSHOT:
+                # Pass through the Snapshot record itself
+                pass_through_records.append(record)
+                if record.get('id'):
+                    snapshot_ids.append(record['id'])
+
+            elif record_type == TYPE_ARCHIVERESULT:
+                # ArchiveResult records: pass through if they have an id
+                if record.get('id'):
+                    pass_through_records.append(record)
+                # If no id, we could create it, but for now just pass through
+                else:
+                    pass_through_records.append(record)
+
+            elif record_type:
+                # Other typed records (Crawl, Tag, etc): pass through
+                pass_through_records.append(record)
+
+            elif record.get('id'):
+                # Untyped record with id - assume it's a snapshot ID
+                snapshot_ids.append(record['id'])
+
+        # Output pass-through records first
+        if not is_tty:
+            for record in pass_through_records:
+                write_record(record)
+
+        if not snapshot_ids:
+            if pass_through_records:
+                rprint(f'[dim]Passed through {len(pass_through_records)} records, no new snapshots to process[/dim]', file=sys.stderr)
+                return 0
+            rprint('[yellow]No valid Snapshot IDs in input[/yellow]', file=sys.stderr)
+            return 1
+
+        snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids))
+
+    if not snapshots:
+        rprint('[yellow]No matching snapshots found[/yellow]', file=sys.stderr)
+        return 0 if pass_through_records else 1
+
+    created_count = 0
+    for snapshot in snapshots:
+        if plugin:
+            # Create for specific plugin only
+            result, created = ArchiveResult.objects.get_or_create(
+                snapshot=snapshot,
+                plugin=plugin,
+                defaults={
+                    'status': status,
+                    'retry_at': timezone.now(),
+                }
+            )
+            if not created and result.status in [ArchiveResult.StatusChoices.FAILED, ArchiveResult.StatusChoices.SKIPPED]:
+                # Reset for retry
+                result.status = status
+                result.retry_at = timezone.now()
+                result.save()
+
+            if not is_tty:
+                write_record(result.to_jsonl())
+            created_count += 1
+        else:
+            # Create all pending plugins
+            snapshot.create_pending_archiveresults()
+            for result in snapshot.archiveresult_set.filter(status=ArchiveResult.StatusChoices.QUEUED):
+                if not is_tty:
+                    write_record(result.to_jsonl())
+                created_count += 1
+
+    rprint(f'[green]Created/queued {created_count} archive results[/green]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+def list_archiveresults(
+    status: Optional[str] = None,
+    plugin: Optional[str] = None,
+    snapshot_id: Optional[str] = None,
+    limit: Optional[int] = None,
+) -> int:
+    """
+    List ArchiveResults as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.core.models import ArchiveResult
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = ArchiveResult.objects.all().order_by('-start_ts')
+
+    # Apply filters
+    filter_kwargs = {
+        'status': status,
+        'plugin': plugin,
+        'snapshot_id': snapshot_id,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for result in queryset:
+        if is_tty:
+            status_color = {
+                'queued': 'yellow',
+                'started': 'blue',
+                'succeeded': 'green',
+                'failed': 'red',
+                'skipped': 'dim',
+                'backoff': 'magenta',
+            }.get(result.status, 'dim')
+            rprint(f'[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}')
+        else:
+            write_record(result.to_jsonl())
+        count += 1
+
+    rprint(f'[dim]Listed {count} archive results[/dim]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# UPDATE
+# =============================================================================
+
+def update_archiveresults(
+    status: Optional[str] = None,
+) -> int:
+    """
+    Update ArchiveResults from stdin JSONL.
+
+    Reads ArchiveResult records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from django.utils import timezone
+
+    from archivebox.misc.jsonl import read_stdin, write_record
+    from archivebox.core.models import ArchiveResult
+
+    is_tty = sys.stdout.isatty()
+
+    records = list(read_stdin())
+    if not records:
+        rprint('[yellow]No records provided via stdin[/yellow]', file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        result_id = record.get('id')
+        if not result_id:
+            continue
+
+        try:
+            result = ArchiveResult.objects.get(id=result_id)
+
+            # Apply updates from CLI flags
+            if status:
+                result.status = status
+                result.retry_at = timezone.now()
+
+            result.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(result.to_jsonl())
+
+        except ArchiveResult.DoesNotExist:
+            rprint(f'[yellow]ArchiveResult not found: {result_id}[/yellow]', file=sys.stderr)
+            continue
+
+    rprint(f'[green]Updated {updated_count} archive results[/green]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# DELETE
+# =============================================================================
+
+def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
+    """
+    Delete ArchiveResults from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
+    from archivebox.core.models import ArchiveResult
+
+    records = list(read_stdin())
+    if not records:
+        rprint('[yellow]No records provided via stdin[/yellow]', file=sys.stderr)
+        return 1
+
+    result_ids = [r.get('id') for r in records if r.get('id')]
+
+    if not result_ids:
+        rprint('[yellow]No valid archive result IDs in input[/yellow]', file=sys.stderr)
+        return 1
+
+    results = ArchiveResult.objects.filter(id__in=result_ids)
+    count = results.count()
+
+    if count == 0:
+        rprint('[yellow]No matching archive results found[/yellow]', file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f'[yellow]Would delete {count} archive results (dry run)[/yellow]', file=sys.stderr)
+        for result in results[:10]:
+            rprint(f'  [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}', file=sys.stderr)
+        if count > 10:
+            rprint(f'  ... and {count - 10} more', file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint('[red]Use --yes to confirm deletion[/red]', file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = results.delete()
+    rprint(f'[green]Deleted {deleted_count} archive results[/green]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+@click.group()
+def main():
+    """Manage ArchiveResult records (plugin extraction results)."""
+    pass
+
+
+@main.command('create')
+@click.option('--snapshot-id', help='Snapshot ID to create results for')
+@click.option('--plugin', '-p', help='Plugin name (e.g., screenshot, singlefile)')
+@click.option('--status', '-s', default='queued', help='Initial status (default: queued)')
+def create_cmd(snapshot_id: Optional[str], plugin: Optional[str], status: str):
+    """Create ArchiveResults for Snapshots from stdin JSONL."""
+    sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
+
+
+@main.command('list')
+@click.option('--status', '-s', help='Filter by status (queued, started, succeeded, failed, skipped)')
+@click.option('--plugin', '-p', help='Filter by plugin name')
+@click.option('--snapshot-id', help='Filter by snapshot ID')
+@click.option('--limit', '-n', type=int, help='Limit number of results')
+def list_cmd(status: Optional[str], plugin: Optional[str],
+             snapshot_id: Optional[str], limit: Optional[int]):
+    """List ArchiveResults as JSONL."""
+    sys.exit(list_archiveresults(
+        status=status,
+        plugin=plugin,
+        snapshot_id=snapshot_id,
+        limit=limit,
+    ))
+
+
+@main.command('update')
+@click.option('--status', '-s', help='Set status')
+def update_cmd(status: Optional[str]):
+    """Update ArchiveResults from stdin JSONL."""
+    sys.exit(update_archiveresults(status=status))
+
+
+@main.command('delete')
+@click.option('--yes', '-y', is_flag=True, help='Confirm deletion')
+@click.option('--dry-run', is_flag=True, help='Show what would be deleted')
+def delete_cmd(yes: bool, dry_run: bool):
+    """Delete ArchiveResults from stdin JSONL."""
+    sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
+
+
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_binary.py

@@ -0,0 +1,290 @@
+#!/usr/bin/env python3
+
+"""
+archivebox binary <action> [args...] [--filters]
+
+Manage Binary records (detected executables like chrome, wget, etc.).
+
+Actions:
+    create  - Create/register a Binary
+    list    - List Binaries as JSONL (with optional filters)
+    update  - Update Binaries from stdin JSONL
+    delete  - Delete Binaries from stdin JSONL
+
+Examples:
+    # List all binaries
+    archivebox binary list
+
+    # List specific binary
+    archivebox binary list --name=chrome
+
+    # List binaries with specific version
+    archivebox binary list --version__icontains=120
+
+    # Delete old binary entries
+    archivebox binary list --name=chrome | archivebox binary delete --yes
+"""
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox binary'
+
+import sys
+from typing import Optional
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+# =============================================================================
+# CREATE
+# =============================================================================
+
+def create_binary(
+    name: str,
+    abspath: str,
+    version: str = '',
+) -> int:
+    """
+    Create/register a Binary.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.machine.models import Binary
+
+    is_tty = sys.stdout.isatty()
+
+    if not name or not abspath:
+        rprint('[red]Both --name and --abspath are required[/red]', file=sys.stderr)
+        return 1
+
+    try:
+        binary, created = Binary.objects.get_or_create(
+            name=name,
+            abspath=abspath,
+            defaults={'version': version}
+        )
+
+        if not is_tty:
+            write_record(binary.to_json())
+
+        if created:
+            rprint(f'[green]Created binary: {name} at {abspath}[/green]', file=sys.stderr)
+        else:
+            rprint(f'[dim]Binary already exists: {name} at {abspath}[/dim]', file=sys.stderr)
+
+        return 0
+
+    except Exception as e:
+        rprint(f'[red]Error creating binary: {e}[/red]', file=sys.stderr)
+        return 1
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+def list_binaries(
+    name: Optional[str] = None,
+    abspath__icontains: Optional[str] = None,
+    version__icontains: Optional[str] = None,
+    limit: Optional[int] = None,
+) -> int:
+    """
+    List Binaries as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.machine.models import Binary
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Binary.objects.all().order_by('name', '-loaded_at')
+
+    # Apply filters
+    filter_kwargs = {
+        'name': name,
+        'abspath__icontains': abspath__icontains,
+        'version__icontains': version__icontains,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for binary in queryset:
+        if is_tty:
+            rprint(f'[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}')
+        else:
+            write_record(binary.to_json())
+        count += 1
+
+    rprint(f'[dim]Listed {count} binaries[/dim]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# UPDATE
+# =============================================================================
+
+def update_binaries(
+    version: Optional[str] = None,
+    abspath: Optional[str] = None,
+) -> int:
+    """
+    Update Binaries from stdin JSONL.
+
+    Reads Binary records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from archivebox.misc.jsonl import read_stdin, write_record
+    from archivebox.machine.models import Binary
+
+    is_tty = sys.stdout.isatty()
+
+    records = list(read_stdin())
+    if not records:
+        rprint('[yellow]No records provided via stdin[/yellow]', file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        binary_id = record.get('id')
+        if not binary_id:
+            continue
+
+        try:
+            binary = Binary.objects.get(id=binary_id)
+
+            # Apply updates from CLI flags
+            if version:
+                binary.version = version
+            if abspath:
+                binary.abspath = abspath
+
+            binary.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(binary.to_json())
+
+        except Binary.DoesNotExist:
+            rprint(f'[yellow]Binary not found: {binary_id}[/yellow]', file=sys.stderr)
+            continue
+
+    rprint(f'[green]Updated {updated_count} binaries[/green]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# DELETE
+# =============================================================================
+
+def delete_binaries(yes: bool = False, dry_run: bool = False) -> int:
+    """
+    Delete Binaries from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
+    from archivebox.machine.models import Binary
+
+    records = list(read_stdin())
+    if not records:
+        rprint('[yellow]No records provided via stdin[/yellow]', file=sys.stderr)
+        return 1
+
+    binary_ids = [r.get('id') for r in records if r.get('id')]
+
+    if not binary_ids:
+        rprint('[yellow]No valid binary IDs in input[/yellow]', file=sys.stderr)
+        return 1
+
+    binaries = Binary.objects.filter(id__in=binary_ids)
+    count = binaries.count()
+
+    if count == 0:
+        rprint('[yellow]No matching binaries found[/yellow]', file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f'[yellow]Would delete {count} binaries (dry run)[/yellow]', file=sys.stderr)
+        for binary in binaries:
+            rprint(f'  {binary.name} {binary.abspath}', file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint('[red]Use --yes to confirm deletion[/red]', file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = binaries.delete()
+    rprint(f'[green]Deleted {deleted_count} binaries[/green]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+@click.group()
+def main():
+    """Manage Binary records (detected executables)."""
+    pass
+
+
+@main.command('create')
+@click.option('--name', '-n', required=True, help='Binary name (e.g., chrome, wget)')
+@click.option('--abspath', '-p', required=True, help='Absolute path to binary')
+@click.option('--version', '-v', default='', help='Binary version')
+def create_cmd(name: str, abspath: str, version: str):
+    """Create/register a Binary."""
+    sys.exit(create_binary(name=name, abspath=abspath, version=version))
+
+
+@main.command('list')
+@click.option('--name', '-n', help='Filter by name')
+@click.option('--abspath__icontains', help='Filter by path contains')
+@click.option('--version__icontains', help='Filter by version contains')
+@click.option('--limit', type=int, help='Limit number of results')
+def list_cmd(name: Optional[str], abspath__icontains: Optional[str],
+             version__icontains: Optional[str], limit: Optional[int]):
+    """List Binaries as JSONL."""
+    sys.exit(list_binaries(
+        name=name,
+        abspath__icontains=abspath__icontains,
+        version__icontains=version__icontains,
+        limit=limit,
+    ))
+
+
+@main.command('update')
+@click.option('--version', '-v', help='Set version')
+@click.option('--abspath', '-p', help='Set path')
+def update_cmd(version: Optional[str], abspath: Optional[str]):
+    """Update Binaries from stdin JSONL."""
+    sys.exit(update_binaries(version=version, abspath=abspath))
+
+
+@main.command('delete')
+@click.option('--yes', '-y', is_flag=True, help='Confirm deletion')
+@click.option('--dry-run', is_flag=True, help='Show what would be deleted')
+def delete_cmd(yes: bool, dry_run: bool):
+    """Delete Binaries from stdin JSONL."""
+    sys.exit(delete_binaries(yes=yes, dry_run=dry_run))
+
+
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_config.py

@@ -1,64 +1,172 @@
 #!/usr/bin/env python3
 
 __package__ = 'archivebox.cli'
-__command__ = 'archivebox config'
 
 import sys
-import argparse
+import rich_click as click
+from rich import print
+from benedict import benedict
 
-from typing import Optional, List, IO
-
-from ..main import config
-from ..util import docstring
-from ..config import OUTPUT_DIR
-from ..logging_util import SmartFormatter, accept_stdin
+from archivebox.misc.util import docstring, enforce_types
+from archivebox.misc.toml_util import CustomTOMLEncoder
 
 
+@enforce_types
+def config(*keys,
+          get: bool=False,
+          set: bool=False, 
+          search: bool=False,
+          reset: bool=False,
+          **kwargs) -> None:
+    """Get and set your ArchiveBox project configuration values"""
+
+    from archivebox.misc.checks import check_data_folder
+    from archivebox.misc.logging_util import printable_config
+    from archivebox.config.collection import load_all_config, write_config_file, get_real_name
+    from archivebox.config.configset import get_flat_config, get_all_configs
+
+    check_data_folder()
+
+    FLAT_CONFIG = get_flat_config()
+    CONFIGS = get_all_configs()
+    
+    config_options: list[str] = list(kwargs.pop('key=value', []) or keys or [f'{key}={val}' for key, val in kwargs.items()])
+    no_args = not (get or set or reset or config_options)
+
+    matching_config = {}
+    if search:
+        if config_options:
+            config_options = [get_real_name(key) for key in config_options]
+            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            for config_section in CONFIGS.values():
+                aliases = getattr(config_section, 'aliases', {})
+                
+                for search_key in config_options:
+                    # search all aliases in the section
+                    for alias_key, key in aliases.items():
+                        if search_key.lower() in alias_key.lower():
+                            matching_config[key] = dict(config_section)[key]
+                    
+                    # search all keys and values in the section
+                    for existing_key, value in dict(config_section).items():
+                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
+                            matching_config[existing_key] = value
+            
+        print(printable_config(matching_config))
+        raise SystemExit(not matching_config)
+
+    elif get or no_args:
+        if config_options:
+            config_options = [get_real_name(key) for key in config_options]
+            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
+            if failed_config:
+                print('\n[red][X] These options failed to get[/red]')
+                print('    {}'.format('\n    '.join(config_options)))
+                raise SystemExit(1)
+        else:
+            matching_config = FLAT_CONFIG
+
+        # Display core config sections
+        for config_section in CONFIGS.values():
+            if hasattr(config_section, 'toml_section_header'):
+                print(f'[grey53]\\[{config_section.toml_section_header}][/grey53]')
+            else:
+                print('[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]')
+
+            kv_in_section = {key: val for key, val in dict(config_section).items() if key in matching_config}
+            print(benedict(kv_in_section).to_toml(encoder=CustomTOMLEncoder()).strip().replace('\n\n', '\n'))
+            print('[grey53]################################################################[/grey53]')
+
+        # Display plugin config section
+        from archivebox.hooks import discover_plugin_configs
+
+        plugin_configs = discover_plugin_configs()
+        plugin_keys = {}
+
+        # Collect all plugin config keys
+        for plugin_name, schema in plugin_configs.items():
+            if 'properties' not in schema:
+                continue
+            for key in schema['properties'].keys():
+                if key in matching_config:
+                    plugin_keys[key] = matching_config[key]
+
+        # Display all plugin config in single [PLUGINS] section
+        if plugin_keys:
+            print(f'[grey53]\\[PLUGINS][/grey53]')
+            print(benedict(plugin_keys).to_toml(encoder=CustomTOMLEncoder()).strip().replace('\n\n', '\n'))
+            print('[grey53]################################################################[/grey53]')
+
+        raise SystemExit(not matching_config)
+
+    elif set:
+        new_config = {}
+        failed_options = []
+        for line in config_options:
+            if line.startswith('#') or not line.strip():
+                continue
+            if '=' not in line:
+                print('[red][X] Config KEY=VALUE must have an = sign in it[/red]')
+                print(f'    {line}')
+                raise SystemExit(2)
+
+            raw_key, val = line.split('=', 1)
+            raw_key = raw_key.upper().strip()
+            key = get_real_name(raw_key)
+            if key != raw_key:
+                print(f'[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]')
+
+            if key in FLAT_CONFIG:
+                new_config[key] = val.strip()
+            else:
+                failed_options.append(line)
+
+        if new_config:
+            before = FLAT_CONFIG
+            matching_config = write_config_file(new_config)
+            after = {**load_all_config(), **get_flat_config()}
+            print(printable_config(matching_config))
+
+            side_effect_changes = {}
+            for key, val in after.items():
+                if key in FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
+                    side_effect_changes[key] = after[key]
+
+            if side_effect_changes:
+                print(file=sys.stderr)
+                print('[yellow][i] Note: This change also affected these other options that depended on it:[/yellow]', file=sys.stderr)
+                print('    {}'.format(printable_config(side_effect_changes, prefix='    ')), file=sys.stderr)
+
+        if failed_options:
+            print()
+            print('[red][X] These options failed to set (check for typos):[/red]')
+            print('    {}'.format('\n    '.join(failed_options)))
+            raise SystemExit(1)
+
+    elif reset:
+        print('[red][X] This command is not implemented yet.[/red]')
+        print('    Please manually remove the relevant lines from your config file:')
+        raise SystemExit(2)
+
+    else:
+        print('[red][X] You must pass either --get or --set, or no arguments to get the whole config.[/red]')
+        print('    archivebox config')
+        print('    archivebox config --get SOME_KEY')
+        print('    archivebox config --set SOME_KEY=SOME_VALUE')
+        raise SystemExit(2)
+
+
+@click.command()
+@click.option('--search', is_flag=True, help='Search config KEYs, VALUEs, and ALIASES for the given term')
+@click.option('--get', is_flag=True, help='Get the value for the given config KEYs')
+@click.option('--set', is_flag=True, help='Set the given KEY=VALUE config values')
+@click.option('--reset', is_flag=True, help='Reset the given KEY config values to their defaults')
+@click.argument('KEY=VALUE', nargs=-1, type=str)
 @docstring(config.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=config.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        '--get', #'-g',
-        action='store_true',
-        help="Get the value for the given config KEYs",
-    )
-    group.add_argument(
-        '--set', #'-s',
-        action='store_true',
-        help="Set the given KEY=VALUE config values",
-    )
-    group.add_argument(
-        '--reset', #'-s',
-        action='store_true',
-        help="Reset the given KEY config values to their defaults",
-    )
-    parser.add_argument(
-        'config_options',
-        nargs='*',
-        type=str,
-        help='KEY or KEY=VALUE formatted config values to get or set',
-    )
-    command = parser.parse_args(args or ())
-
-    config_options_str = ''
-    if not command.config_options:
-        config_options_str = accept_stdin(stdin)
-
-    config(
-        config_options_str=config_options_str,
-        config_options=command.config_options,
-        get=command.get,
-        set=command.set,
-        reset=command.reset,
-        out_dir=pwd or OUTPUT_DIR,
-    )
+def main(**kwargs) -> None:
+    config(**kwargs)
 
 
 if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main()

archivebox/cli/archivebox_crawl.py

@@ -0,0 +1,373 @@
+#!/usr/bin/env python3
+
+"""
+archivebox crawl <action> [args...] [--filters]
+
+Manage Crawl records.
+
+Actions:
+    create  - Create Crawl jobs from URLs
+    list    - List Crawls as JSONL (with optional filters)
+    update  - Update Crawls from stdin JSONL
+    delete  - Delete Crawls from stdin JSONL
+
+Examples:
+    # Create
+    archivebox crawl create https://example.com https://foo.com --depth=1
+    archivebox crawl create --tag=news https://example.com
+
+    # List with filters
+    archivebox crawl list --status=queued
+    archivebox crawl list --urls__icontains=example.com
+
+    # Update
+    archivebox crawl list --status=started | archivebox crawl update --status=queued
+
+    # Delete
+    archivebox crawl list --urls__icontains=spam.com | archivebox crawl delete --yes
+
+    # Full pipeline
+    archivebox crawl create https://example.com | archivebox snapshot create | archivebox run
+"""
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox crawl'
+
+import sys
+from typing import Optional, Iterable
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+# =============================================================================
+# CREATE
+# =============================================================================
+
+def create_crawl(
+    urls: Iterable[str],
+    depth: int = 0,
+    tag: str = '',
+    status: str = 'queued',
+    created_by_id: Optional[int] = None,
+) -> int:
+    """
+    Create a Crawl job from URLs.
+
+    Takes URLs as args or stdin, creates one Crawl with all URLs, outputs JSONL.
+    Pass-through: Records that are not URLs are output unchanged (for piping).
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+
+    created_by_id = created_by_id or get_or_create_system_user_pk()
+    is_tty = sys.stdout.isatty()
+
+    # Collect all input records
+    records = list(read_args_or_stdin(urls))
+
+    if not records:
+        rprint('[yellow]No URLs provided. Pass URLs as arguments or via stdin.[/yellow]', file=sys.stderr)
+        return 1
+
+    # Separate pass-through records from URL records
+    url_list = []
+    pass_through_records = []
+
+    for record in records:
+        record_type = record.get('type', '')
+
+        # Pass-through: output records that aren't URL/Crawl types
+        if record_type and record_type != TYPE_CRAWL and not record.get('url') and not record.get('urls'):
+            pass_through_records.append(record)
+            continue
+
+        # Handle existing Crawl records (just pass through with id)
+        if record_type == TYPE_CRAWL and record.get('id'):
+            pass_through_records.append(record)
+            continue
+
+        # Collect URLs
+        url = record.get('url')
+        if url:
+            url_list.append(url)
+
+        # Handle 'urls' field (newline-separated)
+        urls_field = record.get('urls')
+        if urls_field:
+            for line in urls_field.split('\n'):
+                line = line.strip()
+                if line and not line.startswith('#'):
+                    url_list.append(line)
+
+    # Output pass-through records first
+    if not is_tty:
+        for record in pass_through_records:
+            write_record(record)
+
+    if not url_list:
+        if pass_through_records:
+            # If we had pass-through records but no URLs, that's OK
+            rprint(f'[dim]Passed through {len(pass_through_records)} records, no new URLs[/dim]', file=sys.stderr)
+            return 0
+        rprint('[red]No valid URLs found[/red]', file=sys.stderr)
+        return 1
+
+    try:
+        # Build crawl record with all URLs as newline-separated string
+        crawl_record = {
+            'urls': '\n'.join(url_list),
+            'max_depth': depth,
+            'tags_str': tag,
+            'status': status,
+            'label': '',
+        }
+
+        crawl = Crawl.from_json(crawl_record, overrides={'created_by_id': created_by_id})
+        if not crawl:
+            rprint('[red]Failed to create crawl[/red]', file=sys.stderr)
+            return 1
+
+        # Output JSONL record (only when piped)
+        if not is_tty:
+            write_record(crawl.to_json())
+
+        rprint(f'[green]Created crawl with {len(url_list)} URLs[/green]', file=sys.stderr)
+
+        # If TTY, show human-readable output
+        if is_tty:
+            rprint(f'  [dim]{crawl.id}[/dim]', file=sys.stderr)
+            for url in url_list[:5]:  # Show first 5 URLs
+                rprint(f'    {url[:70]}', file=sys.stderr)
+            if len(url_list) > 5:
+                rprint(f'    ... and {len(url_list) - 5} more', file=sys.stderr)
+
+        return 0
+
+    except Exception as e:
+        rprint(f'[red]Error creating crawl: {e}[/red]', file=sys.stderr)
+        return 1
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+def list_crawls(
+    status: Optional[str] = None,
+    urls__icontains: Optional[str] = None,
+    max_depth: Optional[int] = None,
+    limit: Optional[int] = None,
+) -> int:
+    """
+    List Crawls as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.crawls.models import Crawl
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Crawl.objects.all().order_by('-created_at')
+
+    # Apply filters
+    filter_kwargs = {
+        'status': status,
+        'urls__icontains': urls__icontains,
+        'max_depth': max_depth,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for crawl in queryset:
+        if is_tty:
+            status_color = {
+                'queued': 'yellow',
+                'started': 'blue',
+                'sealed': 'green',
+            }.get(crawl.status, 'dim')
+            url_preview = crawl.urls[:50].replace('\n', ' ')
+            rprint(f'[{status_color}]{crawl.status:8}[/{status_color}] [dim]{crawl.id}[/dim] {url_preview}...')
+        else:
+            write_record(crawl.to_json())
+        count += 1
+
+    rprint(f'[dim]Listed {count} crawls[/dim]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# UPDATE
+# =============================================================================
+
+def update_crawls(
+    status: Optional[str] = None,
+    max_depth: Optional[int] = None,
+) -> int:
+    """
+    Update Crawls from stdin JSONL.
+
+    Reads Crawl records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from django.utils import timezone
+
+    from archivebox.misc.jsonl import read_stdin, write_record
+    from archivebox.crawls.models import Crawl
+
+    is_tty = sys.stdout.isatty()
+
+    records = list(read_stdin())
+    if not records:
+        rprint('[yellow]No records provided via stdin[/yellow]', file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        crawl_id = record.get('id')
+        if not crawl_id:
+            continue
+
+        try:
+            crawl = Crawl.objects.get(id=crawl_id)
+
+            # Apply updates from CLI flags
+            if status:
+                crawl.status = status
+                crawl.retry_at = timezone.now()
+            if max_depth is not None:
+                crawl.max_depth = max_depth
+
+            crawl.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(crawl.to_json())
+
+        except Crawl.DoesNotExist:
+            rprint(f'[yellow]Crawl not found: {crawl_id}[/yellow]', file=sys.stderr)
+            continue
+
+    rprint(f'[green]Updated {updated_count} crawls[/green]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# DELETE
+# =============================================================================
+
+def delete_crawls(yes: bool = False, dry_run: bool = False) -> int:
+    """
+    Delete Crawls from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
+    from archivebox.crawls.models import Crawl
+
+    records = list(read_stdin())
+    if not records:
+        rprint('[yellow]No records provided via stdin[/yellow]', file=sys.stderr)
+        return 1
+
+    crawl_ids = [r.get('id') for r in records if r.get('id')]
+
+    if not crawl_ids:
+        rprint('[yellow]No valid crawl IDs in input[/yellow]', file=sys.stderr)
+        return 1
+
+    crawls = Crawl.objects.filter(id__in=crawl_ids)
+    count = crawls.count()
+
+    if count == 0:
+        rprint('[yellow]No matching crawls found[/yellow]', file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f'[yellow]Would delete {count} crawls (dry run)[/yellow]', file=sys.stderr)
+        for crawl in crawls:
+            url_preview = crawl.urls[:50].replace('\n', ' ')
+            rprint(f'  [dim]{crawl.id}[/dim] {url_preview}...', file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint('[red]Use --yes to confirm deletion[/red]', file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = crawls.delete()
+    rprint(f'[green]Deleted {deleted_count} crawls[/green]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+@click.group()
+def main():
+    """Manage Crawl records."""
+    pass
+
+
+@main.command('create')
+@click.argument('urls', nargs=-1)
+@click.option('--depth', '-d', type=int, default=0, help='Max crawl depth (default: 0)')
+@click.option('--tag', '-t', default='', help='Comma-separated tags to add')
+@click.option('--status', '-s', default='queued', help='Initial status (default: queued)')
+def create_cmd(urls: tuple, depth: int, tag: str, status: str):
+    """Create a Crawl job from URLs or stdin."""
+    sys.exit(create_crawl(urls, depth=depth, tag=tag, status=status))
+
+
+@main.command('list')
+@click.option('--status', '-s', help='Filter by status (queued, started, sealed)')
+@click.option('--urls__icontains', help='Filter by URLs contains')
+@click.option('--max-depth', type=int, help='Filter by max depth')
+@click.option('--limit', '-n', type=int, help='Limit number of results')
+def list_cmd(status: Optional[str], urls__icontains: Optional[str],
+             max_depth: Optional[int], limit: Optional[int]):
+    """List Crawls as JSONL."""
+    sys.exit(list_crawls(
+        status=status,
+        urls__icontains=urls__icontains,
+        max_depth=max_depth,
+        limit=limit,
+    ))
+
+
+@main.command('update')
+@click.option('--status', '-s', help='Set status')
+@click.option('--max-depth', type=int, help='Set max depth')
+def update_cmd(status: Optional[str], max_depth: Optional[int]):
+    """Update Crawls from stdin JSONL."""
+    sys.exit(update_crawls(status=status, max_depth=max_depth))
+
+
+@main.command('delete')
+@click.option('--yes', '-y', is_flag=True, help='Confirm deletion')
+@click.option('--dry-run', is_flag=True, help='Show what would be deleted')
+def delete_cmd(yes: bool, dry_run: bool):
+    """Delete Crawls from stdin JSONL."""
+    sys.exit(delete_crawls(yes=yes, dry_run=dry_run))
+
+
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_extract.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+
+"""
+archivebox extract [snapshot_ids...] [--plugins=NAMES]
+
+Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL.
+
+Input formats:
+    - Snapshot UUIDs (one per line)
+    - JSONL: {"type": "Snapshot", "id": "...", "url": "..."}
+    - JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."}
+
+Output (JSONL):
+    {"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."}
+
+Examples:
+    # Extract specific snapshot
+    archivebox extract 01234567-89ab-cdef-0123-456789abcdef
+
+    # Pipe from snapshot command
+    archivebox snapshot https://example.com | archivebox extract
+
+    # Run specific plugins only
+    archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef
+
+    # Chain commands
+    archivebox crawl https://example.com | archivebox snapshot | archivebox extract
+"""
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox extract'
+
+import sys
+from typing import Optional, List
+
+import rich_click as click
+
+
+def process_archiveresult_by_id(archiveresult_id: str) -> int:
+    """
+    Run extraction for a single ArchiveResult by ID (used by workers).
+
+    Triggers the ArchiveResult's state machine tick() to run the extractor plugin.
+    """
+    from rich import print as rprint
+    from archivebox.core.models import ArchiveResult
+
+    try:
+        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
+    except ArchiveResult.DoesNotExist:
+        rprint(f'[red]ArchiveResult {archiveresult_id} not found[/red]', file=sys.stderr)
+        return 1
+
+    rprint(f'[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]', file=sys.stderr)
+
+    try:
+        # Trigger state machine tick - this runs the actual extraction
+        archiveresult.sm.tick()
+        archiveresult.refresh_from_db()
+
+        if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
+            print(f'[green]Extraction succeeded: {archiveresult.output_str}[/green]')
+            return 0
+        elif archiveresult.status == ArchiveResult.StatusChoices.FAILED:
+            print(f'[red]Extraction failed: {archiveresult.output_str}[/red]', file=sys.stderr)
+            return 1
+        else:
+            # Still in progress or backoff - not a failure
+            print(f'[yellow]Extraction status: {archiveresult.status}[/yellow]')
+            return 0
+
+    except Exception as e:
+        print(f'[red]Extraction error: {type(e).__name__}: {e}[/red]', file=sys.stderr)
+        return 1
+
+
+def run_plugins(
+    args: tuple,
+    plugins: str = '',
+    wait: bool = True,
+) -> int:
+    """
+    Run plugins on Snapshots from input.
+
+    Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from rich import print as rprint
+    from django.utils import timezone
+
+    from archivebox.misc.jsonl import (
+        read_args_or_stdin, write_record,
+        TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
+    )
+    from archivebox.core.models import Snapshot, ArchiveResult
+    from archivebox.workers.orchestrator import Orchestrator
+
+    is_tty = sys.stdout.isatty()
+
+    # Parse comma-separated plugins list once (reused in creation and filtering)
+    plugins_list = [p.strip() for p in plugins.split(',') if p.strip()] if plugins else []
+
+    # Collect all input records
+    records = list(read_args_or_stdin(args))
+
+    if not records:
+        rprint('[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]', file=sys.stderr)
+        return 1
+
+    # Gather snapshot IDs to process
+    snapshot_ids = set()
+    for record in records:
+        record_type = record.get('type')
+
+        if record_type == TYPE_SNAPSHOT:
+            snapshot_id = record.get('id')
+            if snapshot_id:
+                snapshot_ids.add(snapshot_id)
+            elif record.get('url'):
+                # Look up by URL (get most recent if multiple exist)
+                snap = Snapshot.objects.filter(url=record['url']).order_by('-created_at').first()
+                if snap:
+                    snapshot_ids.add(str(snap.id))
+                else:
+                    rprint(f'[yellow]Snapshot not found for URL: {record["url"]}[/yellow]', file=sys.stderr)
+
+        elif record_type == TYPE_ARCHIVERESULT:
+            snapshot_id = record.get('snapshot_id')
+            if snapshot_id:
+                snapshot_ids.add(snapshot_id)
+
+        elif 'id' in record:
+            # Assume it's a snapshot ID
+            snapshot_ids.add(record['id'])
+
+    if not snapshot_ids:
+        rprint('[red]No valid snapshot IDs found in input[/red]', file=sys.stderr)
+        return 1
+
+    # Get snapshots and ensure they have pending ArchiveResults
+    processed_count = 0
+    for snapshot_id in snapshot_ids:
+        try:
+            snapshot = Snapshot.objects.get(id=snapshot_id)
+        except Snapshot.DoesNotExist:
+            rprint(f'[yellow]Snapshot {snapshot_id} not found[/yellow]', file=sys.stderr)
+            continue
+
+        # Create pending ArchiveResults if needed
+        if plugins_list:
+            # Only create for specific plugins
+            for plugin_name in plugins_list:
+                result, created = ArchiveResult.objects.get_or_create(
+                    snapshot=snapshot,
+                    plugin=plugin_name,
+                    defaults={
+                        'status': ArchiveResult.StatusChoices.QUEUED,
+                        'retry_at': timezone.now(),
+                    }
+                )
+                if not created and result.status in [ArchiveResult.StatusChoices.FAILED, ArchiveResult.StatusChoices.SKIPPED]:
+                    # Reset for retry
+                    result.status = ArchiveResult.StatusChoices.QUEUED
+                    result.retry_at = timezone.now()
+                    result.save()
+        else:
+            # Create all pending plugins
+            snapshot.create_pending_archiveresults()
+
+        # Reset snapshot status to allow processing
+        if snapshot.status == Snapshot.StatusChoices.SEALED:
+            snapshot.status = Snapshot.StatusChoices.STARTED
+            snapshot.retry_at = timezone.now()
+            snapshot.save()
+
+        processed_count += 1
+
+    if processed_count == 0:
+        rprint('[red]No snapshots to process[/red]', file=sys.stderr)
+        return 1
+
+    rprint(f'[blue]Queued {processed_count} snapshots for extraction[/blue]', file=sys.stderr)
+
+    # Run orchestrator if --wait (default)
+    if wait:
+        rprint('[blue]Running plugins...[/blue]', file=sys.stderr)
+        orchestrator = Orchestrator(exit_on_idle=True)
+        orchestrator.runloop()
+
+    # Output results as JSONL (when piped) or human-readable (when TTY)
+    for snapshot_id in snapshot_ids:
+        try:
+            snapshot = Snapshot.objects.get(id=snapshot_id)
+            results = snapshot.archiveresult_set.all()
+            if plugins_list:
+                results = results.filter(plugin__in=plugins_list)
+
+            for result in results:
+                if is_tty:
+                    status_color = {
+                        'succeeded': 'green',
+                        'failed': 'red',
+                        'skipped': 'yellow',
+                    }.get(result.status, 'dim')
+                    rprint(f'  [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ""}', file=sys.stderr)
+                else:
+                    write_record(result.to_jsonl())
+        except Snapshot.DoesNotExist:
+            continue
+
+    return 0
+
+
+def is_archiveresult_id(value: str) -> bool:
+    """Check if value looks like an ArchiveResult UUID."""
+    import re
+    uuid_pattern = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', re.I)
+    if not uuid_pattern.match(value):
+        return False
+    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
+    from archivebox.core.models import ArchiveResult
+    return ArchiveResult.objects.filter(id=value).exists()
+
+
+@click.command()
+@click.option('--plugins', '-p', default='', help='Comma-separated list of plugins to run (e.g., screenshot,singlefile)')
+@click.option('--wait/--no-wait', default=True, help='Wait for plugins to complete (default: wait)')
+@click.argument('args', nargs=-1)
+def main(plugins: str, wait: bool, args: tuple):
+    """Run plugins on Snapshots, or process existing ArchiveResults by ID"""
+    from archivebox.misc.jsonl import read_args_or_stdin
+
+    # Read all input
+    records = list(read_args_or_stdin(args))
+
+    if not records:
+        from rich import print as rprint
+        rprint('[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]', file=sys.stderr)
+        sys.exit(1)
+
+    # Check if input looks like existing ArchiveResult IDs to process
+    all_are_archiveresult_ids = all(
+        is_archiveresult_id(r.get('id') or r.get('url', ''))
+        for r in records
+    )
+
+    if all_are_archiveresult_ids:
+        # Process existing ArchiveResults by ID
+        exit_code = 0
+        for record in records:
+            archiveresult_id = record.get('id') or record.get('url')
+            result = process_archiveresult_by_id(archiveresult_id)
+            if result != 0:
+                exit_code = result
+        sys.exit(exit_code)
+    else:
+        # Default behavior: run plugins on Snapshots from input
+        sys.exit(run_plugins(args, plugins=plugins, wait=wait))
+
+
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_help.py

@@ -1,32 +1,105 @@
 #!/usr/bin/env python3
-
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox help'
 
-import sys
-import argparse
+import os    
+from pathlib import Path
 
-from typing import Optional, List, IO
-
-from ..main import help
-from ..util import docstring
-from ..config import OUTPUT_DIR
-from ..logging_util import SmartFormatter, reject_stdin
+import click
+from rich import print
+from rich.panel import Panel
 
 
-@docstring(help.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=help.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.parse_args(args or ())
-    reject_stdin(__command__, stdin)
+def help() -> None:
+    """Print the ArchiveBox help message and usage"""
+
+    from archivebox.cli import ArchiveBoxGroup
+    from archivebox.config import CONSTANTS
+    from archivebox.config.permissions import IN_DOCKER
+    from archivebox.misc.logging_util import log_cli_command
     
-    help(out_dir=pwd or OUTPUT_DIR)
+    log_cli_command('help', [], None, '.')
+    
+    COMMANDS_HELP_TEXT = '\n    '.join(
+        f'[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}'
+        for cmd in ArchiveBoxGroup.meta_commands.keys()
+    ) + '\n\n    ' + '\n    '.join(
+        f'[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}'
+        for cmd in ArchiveBoxGroup.setup_commands.keys()
+    ) + '\n\n    ' + '\n    '.join(
+        f'[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}'
+        for cmd in ArchiveBoxGroup.archive_commands.keys()
+    )
+    
+    DOCKER_USAGE = '''
+[dodger_blue3]Docker Usage:[/dodger_blue3]
+    [grey53]# using Docker Compose:[/grey53]
+    [blue]docker compose run[/blue] [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
 
+    [grey53]# using Docker:[/grey53]
+    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 8000:8000[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
+''' if IN_DOCKER else ''
+    DOCKER_DOCS = '\n    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker[/link]' if IN_DOCKER else ''
+    DOCKER_OUTSIDE_HINT = "\n    [grey53]# outside of Docker:[/grey53]" if IN_DOCKER else ''
+    DOCKER_CMD_PREFIX = "[blue]docker ... [/blue]" if IN_DOCKER else ''
+
+    print(f'''{DOCKER_USAGE}
+[deep_sky_blue4]Usage:[/deep_sky_blue4]{DOCKER_OUTSIDE_HINT}
+    [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
+
+[deep_sky_blue4]Commands:[/deep_sky_blue4]
+    {COMMANDS_HELP_TEXT}
+
+[deep_sky_blue4]Documentation:[/deep_sky_blue4]
+    [link=https://github.com/ArchiveBox/ArchiveBox/wiki]https://github.com/ArchiveBox/ArchiveBox/wiki[/link]{DOCKER_DOCS}
+    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage[/link]
+    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
+''')
+    
+    
+    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
+        pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path('~').expanduser()), '~')
+        EXAMPLE_USAGE = f'''
+[light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]
+
+[violet]Hint:[/violet] [i]Common maintenance tasks:[/i]
+    [dark_green]archivebox[/dark_green] [green]init[/green]      [grey53]# make sure database is up-to-date (safe to run multiple times)[/grey53]
+    [dark_green]archivebox[/dark_green] [green]install[/green]   [grey53]# make sure plugins are up-to-date (wget, chrome, singlefile, etc.)[/grey53]
+    [dark_green]archivebox[/dark_green] [green]status[/green]    [grey53]# get a health checkup report on your collection[/grey53]
+    [dark_green]archivebox[/dark_green] [green]update[/green]    [grey53]# retry any previously failed or interrupted archiving tasks[/grey53]
+
+[violet]Hint:[/violet] [i]More example usage:[/i]
+    [dark_green]archivebox[/dark_green] [green]add[/green] --depth=1 "https://example.com/some/page"
+    [dark_green]archivebox[/dark_green] [green]list[/green] --sort=timestamp --csv=timestamp,downloaded_at,url,title
+    [dark_green]archivebox[/dark_green] [green]schedule[/green] --every=day --depth=1 "https://example.com/some/feed.rss"
+    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:8000[/blue]                [grey53]# Start the Web UI / API server[/grey53]
+'''
+        print(Panel(EXAMPLE_USAGE, expand=False, border_style='grey53', title='[green3]:white_check_mark: A collection [light_slate_blue]DATA DIR[/light_slate_blue] is currently active[/green3]', subtitle='Commands run inside this dir will only apply to this collection.'))
+    else:
+        DATA_SETUP_HELP = '\n'
+        if IN_DOCKER:
+            DATA_SETUP_HELP += '[violet]Hint:[/violet] When using Docker, you need to mount a volume to use as your data dir:\n'
+            DATA_SETUP_HELP += '    docker run [violet]-v /some/path/data:/data[/violet] archivebox/archivebox ...\n\n'
+        DATA_SETUP_HELP += 'To load an [dark_blue]existing[/dark_blue] collection:\n'
+        DATA_SETUP_HELP += '    1. [green]cd[/green] ~/archivebox/data     [grey53]# go into existing [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n'
+        DATA_SETUP_HELP += f'    2. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# migrate to latest version (safe to run multiple times)[/grey53]\n'
+        DATA_SETUP_HELP += f'    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-update all plugins (wget, chrome, singlefile, etc.)[/grey53]\n'
+        DATA_SETUP_HELP += f'    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ...get help with next steps... [/grey53]\n\n'
+        DATA_SETUP_HELP += 'To start a [sea_green1]new[/sea_green1] collection:\n'
+        DATA_SETUP_HELP += '    1. [green]mkdir[/green] ~/archivebox/data  [grey53]# create a new, empty [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n'
+        DATA_SETUP_HELP += '    2. [green]cd[/green] ~/archivebox/data     [grey53]# cd into the new directory[/grey53]\n'
+        DATA_SETUP_HELP += f'    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# initialize ArchiveBox in the new data dir[/grey53]\n'
+        DATA_SETUP_HELP += f'    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-install all plugins (wget, chrome, singlefile, etc.)[/grey53]\n'
+        DATA_SETUP_HELP += f'    5. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ... get help with next steps... [/grey53]\n'
+        print(Panel(DATA_SETUP_HELP, expand=False, border_style='grey53', title='[red]:cross_mark: No collection is currently active[/red]', subtitle='All archivebox [green]commands[/green] should be run from inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]'))
+
+
+
+@click.command()
+@click.option('--help', '-h', is_flag=True, help='Show help')
+def main(**kwargs):
+    """Print the ArchiveBox help message and usage"""
+    return help()
 
 if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main()

archivebox/cli/archivebox_init.py

@@ -1,52 +1,195 @@
 #!/usr/bin/env python3
 
 __package__ = 'archivebox.cli'
-__command__ = 'archivebox init'
 
+import os
 import sys
-import argparse
+from pathlib import Path
 
-from typing import Optional, List, IO
+from rich import print
+import rich_click as click
 
-from ..main import init
-from ..util import docstring
-from ..config import OUTPUT_DIR
-from ..logging_util import SmartFormatter, reject_stdin
+from archivebox.misc.util import docstring, enforce_types
 
 
-@docstring(init.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=init.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        '--force', # '-f',
-        action='store_true',
-        help='Ignore unrecognized files in current directory and initialize anyway',
-    )
-    parser.add_argument(
-        '--quick', '-q',
-        action='store_true',
-        help='Run any updates or migrations without rechecking all snapshot dirs',
-    )
-    parser.add_argument(
-        '--setup', #'-s',
-        action='store_true',
-        help='Automatically install dependencies and extras used for archiving',
-    )
-    command = parser.parse_args(args or ())
-    reject_stdin(__command__, stdin)
-
-    init(
-        force=command.force,
-        quick=command.quick,
-        setup=command.setup,
-        out_dir=pwd or OUTPUT_DIR,
-    )
+@enforce_types
+def init(force: bool=False, quick: bool=False, install: bool=False) -> None:
+    """Initialize a new ArchiveBox collection in the current directory"""
     
+    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
+    from archivebox.config.common import SERVER_CONFIG
+    from archivebox.config.collection import write_config_file
+    from archivebox.misc.legacy import parse_json_main_index, parse_json_links_details, SnapshotDict
+    from archivebox.misc.db import apply_migrations
+    
+    # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
+    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
+    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
+
+    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
+    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
+    if is_empty and not existing_index:
+        print(f'[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]')
+        print('[green]----------------------------------------------------------------------[/green]')
+    elif existing_index:
+        # TODO: properly detect and print the existing version in current index as well
+        print(f'[green][*] Verifying and updating existing ArchiveBox collection to v{VERSION}...[/green]')
+        print('[green]----------------------------------------------------------------------[/green]')
+    else:
+        if force:
+            print('[red][!] This folder appears to already have files in it, but no index.sqlite3 is present.[/red]')
+            print('[red]    Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).[/red]')
+        else:
+            print(
+                ("[red][X] This folder appears to already have files in it, but no index.sqlite3 present.[/red]\n\n"
+                "    You must run init in a completely empty directory, or an existing data folder.\n\n"
+                "    [violet]Hint:[/violet] To import an existing data folder make sure to cd into the folder first, \n"
+                "    then run and run 'archivebox init' to pick up where you left off.\n\n"
+                "    (Always make sure your data folder is backed up first before updating ArchiveBox)"
+                )
+            )
+            raise SystemExit(2)
+
+    if existing_index:
+        print('\n[green][*] Verifying archive folder structure...[/green]')
+    else:
+        print('\n[green][+] Building archive folder structure...[/green]')
+    
+    print(f'    + ./{CONSTANTS.ARCHIVE_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.SOURCES_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.LOGS_DIR.relative_to(DATA_DIR)}...')
+    Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
+    Path(CONSTANTS.ARCHIVE_DIR).mkdir(exist_ok=True)
+    Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
+    
+    print(f'    + ./{CONSTANTS.CONFIG_FILE.relative_to(DATA_DIR)}...')
+    
+    # create the .archivebox_id file with a unique ID for this collection
+    from archivebox.config.paths import _get_collection_id
+    _get_collection_id(DATA_DIR, force_create=True)
+    
+    # create the ArchiveBox.conf file
+    write_config_file({'SECRET_KEY': SERVER_CONFIG.SECRET_KEY})
+
+
+    if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
+        print('\n[green][*] Verifying main SQL index and running any migrations needed...[/green]')
+    else:
+        print('\n[green][+] Building main SQL index and running initial migrations...[/green]')
+    
+    from archivebox.config.django import setup_django
+    setup_django()
+    
+    for migration_line in apply_migrations(DATA_DIR):
+        sys.stdout.write(f'    {migration_line}\n')
+
+    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
+    print()
+    print(f'    √ ./{CONSTANTS.DATABASE_FILE.relative_to(DATA_DIR)}')
+    
+    # from django.contrib.auth.models import User
+    # if SHELL_CONFIG.IS_TTY and not User.objects.filter(is_superuser=True).exclude(username='system').exists():
+    #     print('{green}[+] Creating admin user account...{reset}'.format(**SHELL_CONFIG.ANSI))
+    #     call_command("createsuperuser", interactive=True)
+
+    print()
+    print('[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]')
+
+    from archivebox.core.models import Snapshot
+
+    all_links = Snapshot.objects.none()
+    pending_links: dict[str, SnapshotDict] = {}
+
+    if existing_index:
+        all_links = Snapshot.objects.all()
+        print(f'    √ Loaded {all_links.count()} links from existing main index.')
+
+    if quick:
+        print('    > Skipping orphan snapshot import (quick mode)')
+    else:
+        try:
+            # Import orphaned links from legacy JSON indexes
+            orphaned_json_links = {
+                link_dict['url']: link_dict
+                for link_dict in parse_json_main_index(DATA_DIR)
+                if not all_links.filter(url=link_dict['url']).exists()
+            }
+            if orphaned_json_links:
+                pending_links.update(orphaned_json_links)
+                print(f'    [yellow]√ Added {len(orphaned_json_links)} orphaned links from existing JSON index...[/yellow]')
+
+            orphaned_data_dir_links = {
+                link_dict['url']: link_dict
+                for link_dict in parse_json_links_details(DATA_DIR)
+                if not all_links.filter(url=link_dict['url']).exists()
+            }
+            if orphaned_data_dir_links:
+                pending_links.update(orphaned_data_dir_links)
+                print(f'    [yellow]√ Added {len(orphaned_data_dir_links)} orphaned links from existing archive directories.[/yellow]')
+
+            if pending_links:
+                for link_dict in pending_links.values():
+                    Snapshot.from_json(link_dict)
+
+            # Hint for orphaned snapshot directories
+            print()
+            print('    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:')
+            print('        archivebox update')
+
+        except (KeyboardInterrupt, SystemExit):
+            print(file=sys.stderr)
+            print('[yellow]:stop_sign: Stopped checking archive directories due to Ctrl-C/SIGTERM[/yellow]', file=sys.stderr)
+            print('    Your archive data is safe, but you should re-run `archivebox init` to finish the process later.', file=sys.stderr)
+            print(file=sys.stderr)
+            print('    [violet]Hint:[/violet] In the future you can run a quick init without checking dirs like so:', file=sys.stderr)
+            print('        archivebox init --quick', file=sys.stderr)
+            raise SystemExit(1)
+
+    print('\n[green]----------------------------------------------------------------------[/green]')
+
+    from django.contrib.auth.models import User
+
+    if (SERVER_CONFIG.ADMIN_USERNAME and SERVER_CONFIG.ADMIN_PASSWORD) and not User.objects.filter(username=SERVER_CONFIG.ADMIN_USERNAME).exists():
+        print('[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]')
+        User.objects.create_superuser(username=SERVER_CONFIG.ADMIN_USERNAME, password=SERVER_CONFIG.ADMIN_PASSWORD)
+
+    if existing_index:
+        print('[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]')
+    else:
+        print(f'[green][√] Done. A new ArchiveBox collection was initialized ({len(all_links) + len(pending_links)} links).[/green]')
+
+    
+    CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.DEFAULT_LIB_DIR.mkdir(parents=True, exist_ok=True)
+    
+    from archivebox.config.common import STORAGE_CONFIG
+    STORAGE_CONFIG.TMP_DIR.mkdir(parents=True, exist_ok=True)
+    STORAGE_CONFIG.LIB_DIR.mkdir(parents=True, exist_ok=True)
+    
+    if install:
+        from archivebox.cli.archivebox_install import install as install_method
+        install_method()
+
+    if Snapshot.objects.count() < 25:     # hide the hints for experienced users
+        print()
+        print('    [violet]Hint:[/violet] To view your archive index, run:')
+        print('        archivebox server  # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]')
+        print()
+        print('    To add new links, you can run:')
+        print("        archivebox add < ~/some/path/to/list_of_links.txt")
+        print()
+        print('    For more usage and examples, run:')
+        print('        archivebox help')
+
+
+
+@click.command()
+@click.option('--force', '-f', is_flag=True, help='Ignore unrecognized files in current directory and initialize anyway')
+@click.option('--quick', '-q', is_flag=True, help='Run any updates or migrations without rechecking all snapshot dirs')
+@click.option('--install', '-s', is_flag=True, help='Automatically install dependencies and extras used for archiving')
+@docstring(init.__doc__)
+def main(**kwargs) -> None:
+    init(**kwargs)
+
 
 if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main()

archivebox/cli/archivebox_install.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+
+__package__ = 'archivebox.cli'
+
+import os
+import sys
+import shutil
+
+import rich_click as click
+from rich import print
+
+from archivebox.misc.util import docstring, enforce_types
+
+
+@enforce_types
+def install(dry_run: bool=False) -> None:
+    """Detect and install ArchiveBox dependencies by running a dependency-check crawl"""
+
+    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
+    from archivebox.config.paths import ARCHIVE_DIR
+    from archivebox.misc.logging import stderr
+    from archivebox.cli.archivebox_init import init
+
+    if not (os.access(ARCHIVE_DIR, os.R_OK) and ARCHIVE_DIR.is_dir()):
+        init()  # must init full index because we need a db to store Binary entries in
+
+    print('\n[green][+] Detecting ArchiveBox dependencies...[/green]')
+
+    if IS_ROOT:
+        EUID = os.geteuid()
+        print()
+        print(f'[yellow]:warning:  Running as UID=[blue]{EUID}[/blue].[/yellow]')
+        print(f'    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].')
+        print()
+
+    if dry_run:
+        print('[dim]Dry run - would create a crawl to detect dependencies[/dim]')
+        return
+
+    # Set up Django
+    from archivebox.config.django import setup_django
+    setup_django()
+
+    from django.utils import timezone
+    from archivebox.crawls.models import Crawl
+    from archivebox.base_models.models import get_or_create_system_user_pk
+
+    # Create a crawl for dependency detection
+    # Using a minimal crawl that will trigger on_Crawl hooks
+    created_by_id = get_or_create_system_user_pk()
+
+    crawl, created = Crawl.objects.get_or_create(
+        urls='archivebox://install',
+        defaults={
+            'label': 'Dependency detection',
+            'created_by_id': created_by_id,
+            'max_depth': 0,
+            'status': 'queued',
+        }
+    )
+
+    # If crawl already existed, reset it to queued state so it can be processed again
+    if not created:
+        crawl.status = 'queued'
+        crawl.retry_at = timezone.now()
+        crawl.save()
+
+    print(f'[+] Created dependency detection crawl: {crawl.id}')
+    print(f'[+] Crawl status: {crawl.status}, retry_at: {crawl.retry_at}')
+
+    # Verify the crawl is in the queue
+    from archivebox.crawls.models import Crawl as CrawlModel
+    queued_crawls = CrawlModel.objects.filter(
+        retry_at__lte=timezone.now()
+    ).exclude(
+        status__in=CrawlModel.FINAL_STATES
+    )
+    print(f'[+] Crawls in queue: {queued_crawls.count()}')
+    if queued_crawls.exists():
+        for c in queued_crawls:
+            print(f'    - Crawl {c.id}: status={c.status}, retry_at={c.retry_at}')
+
+    print('[+] Running crawl to detect binaries via on_Crawl hooks...')
+    print()
+
+    # Run the crawl synchronously (this triggers on_Crawl hooks)
+    from archivebox.workers.orchestrator import Orchestrator
+    orchestrator = Orchestrator(exit_on_idle=True)
+    orchestrator.runloop()
+
+    print()
+
+    # Check for superuser
+    from django.contrib.auth import get_user_model
+    User = get_user_model()
+
+    if not User.objects.filter(is_superuser=True).exclude(username='system').exists():
+        stderr('\n[+] Don\'t forget to create a new admin user for the Web UI...', color='green')
+        stderr('    archivebox manage createsuperuser')
+
+    print()
+
+    # Run version to show full status
+    archivebox_path = shutil.which('archivebox') or sys.executable
+    if 'python' in archivebox_path:
+        os.system(f'{sys.executable} -m archivebox version')
+    else:
+        os.system(f'{archivebox_path} version')
+
+
+@click.command()
+@click.option('--dry-run', '-d', is_flag=True, help='Show what would happen without actually running', default=False)
+@docstring(install.__doc__)
+def main(**kwargs) -> None:
+    install(**kwargs)
+    
+
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_list.py

@@ -1,139 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = 'archivebox.cli'
-__command__ = 'archivebox list'
-
-import sys
-import argparse
-
-from typing import Optional, List, IO
-
-from ..main import list_all
-from ..util import docstring
-from ..config import OUTPUT_DIR
-from ..index import (
-    LINK_FILTERS,
-    get_indexed_folders,
-    get_archived_folders,
-    get_unarchived_folders,
-    get_present_folders,
-    get_valid_folders,
-    get_invalid_folders,
-    get_duplicate_folders,
-    get_orphaned_folders,
-    get_corrupted_folders,
-    get_unrecognized_folders,
-)
-from ..logging_util import SmartFormatter, reject_stdin, stderr
-
-
-@docstring(list_all.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=list_all.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        '--csv', #'-c',
-        type=str,
-        help="Print the output in CSV format with the given columns, e.g.: timestamp,url,extension",
-        default=None,
-    )
-    group.add_argument(
-        '--json', #'-j',
-        action='store_true',
-        help="Print the output in JSON format with all columns included",
-    )
-    group.add_argument(
-        '--html',
-        action='store_true',
-        help="Print the output in HTML format"
-    )
-    parser.add_argument(
-        '--with-headers',
-        action='store_true',
-        help='Include the headers in the output document' 
-    )
-    parser.add_argument(
-        '--sort', #'-s',
-        type=str,
-        help="List the links sorted using the given key, e.g. timestamp or updated",
-        default=None,
-    )
-    parser.add_argument(
-        '--before', #'-b',
-        type=float,
-        help="List only links bookmarked before (less than) the given timestamp",
-        default=None,
-    )
-    parser.add_argument(
-        '--after', #'-a',
-        type=float,
-        help="List only links bookmarked after (greater than or equal to) the given timestamp",
-        default=None,
-    )
-    parser.add_argument(
-        '--status',
-        type=str,
-        choices=('indexed', 'archived', 'unarchived', 'present', 'valid', 'invalid', 'duplicate', 'orphaned', 'corrupted', 'unrecognized'),
-        default='indexed',
-        help=(
-            'List only links or data directories that have the given status\n'
-            f'    indexed       {get_indexed_folders.__doc__} (the default)\n'
-            f'    archived      {get_archived_folders.__doc__}\n'
-            f'    unarchived    {get_unarchived_folders.__doc__}\n'
-            '\n'
-            f'    present       {get_present_folders.__doc__}\n'
-            f'    valid         {get_valid_folders.__doc__}\n'
-            f'    invalid       {get_invalid_folders.__doc__}\n'
-            '\n'
-            f'    duplicate     {get_duplicate_folders.__doc__}\n'
-            f'    orphaned      {get_orphaned_folders.__doc__}\n'
-            f'    corrupted     {get_corrupted_folders.__doc__}\n'
-            f'    unrecognized  {get_unrecognized_folders.__doc__}\n'
-        )
-    )
-    parser.add_argument(
-        '--filter-type', '-t',
-        type=str,
-        choices=(*LINK_FILTERS.keys(), 'search'),
-        default='exact',
-        help='Type of pattern matching to use when filtering URLs',
-    )
-    parser.add_argument(
-        'filter_patterns',
-        nargs='*',
-        type=str,
-        default=None,
-        help='List only URLs matching these filter patterns'
-    )
-    command = parser.parse_args(args or ())
-    reject_stdin(stdin)
-
-    if command.with_headers and not (command.json or command.html or command.csv):
-        stderr(
-            '[X] --with-headers can only be used with --json, --html or --csv options\n',
-            color='red',
-        )
-        raise SystemExit(2)
-
-    matching_folders = list_all(
-        filter_patterns=command.filter_patterns,
-        filter_type=command.filter_type,
-        status=command.status,
-        after=command.after,
-        before=command.before,
-        sort=command.sort,
-        csv=command.csv,
-        json=command.json,
-        html=command.html,
-        with_headers=command.with_headers,
-        out_dir=pwd or OUTPUT_DIR,
-    )
-    raise SystemExit(not matching_folders)
-
-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/cli/archivebox_machine.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+
+"""
+archivebox machine <action> [--filters]
+
+Manage Machine records (system-managed, mostly read-only).
+
+Machine records track the host machines where ArchiveBox runs.
+They are created automatically by the system and are primarily for debugging.
+
+Actions:
+    list    - List Machines as JSONL (with optional filters)
+
+Examples:
+    # List all machines
+    archivebox machine list
+
+    # List machines by hostname
+    archivebox machine list --hostname__icontains=myserver
+"""
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox machine'
+
+import sys
+from typing import Optional
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+def list_machines(
+    hostname__icontains: Optional[str] = None,
+    os_platform: Optional[str] = None,
+    limit: Optional[int] = None,
+) -> int:
+    """
+    List Machines as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.machine.models import Machine
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Machine.objects.all().order_by('-created_at')
+
+    # Apply filters
+    filter_kwargs = {
+        'hostname__icontains': hostname__icontains,
+        'os_platform': os_platform,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for machine in queryset:
+        if is_tty:
+            rprint(f'[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}')
+        else:
+            write_record(machine.to_json())
+        count += 1
+
+    rprint(f'[dim]Listed {count} machines[/dim]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+@click.group()
+def main():
+    """Manage Machine records (read-only, system-managed)."""
+    pass
+
+
+@main.command('list')
+@click.option('--hostname__icontains', help='Filter by hostname contains')
+@click.option('--os-platform', help='Filter by OS platform')
+@click.option('--limit', '-n', type=int, help='Limit number of results')
+def list_cmd(hostname__icontains: Optional[str], os_platform: Optional[str], limit: Optional[int]):
+    """List Machines as JSONL."""
+    sys.exit(list_machines(
+        hostname__icontains=hostname__icontains,
+        os_platform=os_platform,
+        limit=limit,
+    ))
+
+
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_manage.py

@@ -1,24 +1,33 @@
 #!/usr/bin/env python3
 
 __package__ = 'archivebox.cli'
-__command__ = 'archivebox manage'
 
-import sys
-
-from typing import Optional, List, IO
-
-from ..main import manage
-from ..util import docstring
-from ..config import OUTPUT_DIR
+import rich_click as click
+from archivebox.misc.util import docstring, enforce_types
 
 
+@enforce_types
+def manage(args: list[str] | None=None) -> None:
+    """Run an ArchiveBox Django management command"""
+
+    from archivebox.config.common import SHELL_CONFIG
+    from archivebox.misc.logging import stderr
+
+    if (args and "createsuperuser" in args) and (SHELL_CONFIG.IN_DOCKER and not SHELL_CONFIG.IS_TTY):
+        stderr('[!] Warning: you need to pass -it to use interactive commands in docker', color='lightyellow')
+        stderr('    docker run -it archivebox manage {}'.format(' '.join(args or ['...'])), color='lightyellow')
+        stderr('')
+
+    from django.core.management import execute_from_command_line
+    execute_from_command_line(['manage.py', *(args or ['help'])])
+
+
+@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
+@click.argument('args', nargs=-1)
 @docstring(manage.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    manage(
-        args=args,
-        out_dir=pwd or OUTPUT_DIR,
-    )
+def main(args: list[str] | None=None) -> None:
+    manage(args=args)
 
 
 if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main()

archivebox/cli/archivebox_mcp.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""
+archivebox mcp
+
+Start the Model Context Protocol (MCP) server in stdio mode.
+Exposes all ArchiveBox CLI commands as MCP tools for AI agents.
+"""
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox mcp'
+
+import rich_click as click
+
+from archivebox.misc.util import docstring, enforce_types
+
+
+@enforce_types
+def mcp():
+    """
+    Start the MCP server in stdio mode for AI agent control.
+
+    The MCP (Model Context Protocol) server exposes all ArchiveBox CLI commands
+    as tools that AI agents can discover and execute. It communicates via JSON-RPC
+    2.0 over stdin/stdout.
+
+    Example usage with an MCP client:
+        archivebox mcp < requests.jsonl > responses.jsonl
+
+    Or interactively:
+        archivebox mcp
+        {"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}
+        {"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}
+    """
+
+    from mcp.server import run_mcp_server
+
+    # Run the stdio server (blocks until stdin closes)
+    run_mcp_server()
+
+
+@click.command()
+@docstring(mcp.__doc__)
+def main(**kwargs):
+    """Start the MCP server in stdio mode"""
+    mcp()
+
+
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_oneshot.py

@@ -1,73 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = 'archivebox.cli'
-__command__ = 'archivebox oneshot'
-
-import sys
-import argparse
-
-from pathlib import Path
-from typing import List, Optional, IO
-
-from ..main import oneshot
-from ..util import docstring
-from ..config import OUTPUT_DIR
-from ..logging_util import SmartFormatter, accept_stdin, stderr
-
-
-@docstring(oneshot.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=oneshot.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        'url',
-        type=str,
-        default=None,
-        help=(
-            'URLs or paths to archive e.g.:\n'
-            '    https://getpocket.com/users/USERNAME/feed/all\n'
-            '    https://example.com/some/rss/feed.xml\n'
-            '    https://example.com\n'
-            '    ~/Downloads/firefox_bookmarks_export.html\n'
-            '    ~/Desktop/sites_list.csv\n'
-        )
-    )
-    parser.add_argument(
-        "--extract",
-        type=str,
-        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
-              This does not take precedence over the configuration",
-        default=""
-    )
-    parser.add_argument(
-        '--out-dir',
-        type=str,
-        default=OUTPUT_DIR,
-        help= "Path to save the single archive folder to, e.g. ./example.com_archive"
-    )
-    command = parser.parse_args(args or ())
-    stdin_url = None
-    url = command.url
-    if not url:
-        stdin_url = accept_stdin(stdin)
-
-    if (stdin_url and url) or (not stdin and not url):
-        stderr(
-            '[X] You must pass a URL/path to add via stdin or CLI arguments.\n',
-            color='red',
-        )
-        raise SystemExit(2)
-    
-    oneshot(
-        url=stdin_url or url,
-        out_dir=Path(command.out_dir).resolve(),
-        extractors=command.extract,
-    )
-
-
-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/cli/archivebox_orchestrator.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+"""
+archivebox orchestrator [--daemon]
+
+Start the orchestrator process that manages workers.
+
+The orchestrator polls queues for each model type (Crawl, Snapshot, ArchiveResult)
+and lazily spawns worker processes when there is work to be done.
+"""
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox orchestrator'
+
+import sys
+
+import rich_click as click
+
+from archivebox.misc.util import docstring
+
+
+def orchestrator(daemon: bool = False, watch: bool = False) -> int:
+    """
+    Start the orchestrator process.
+    
+    The orchestrator:
+    1. Polls each model queue (Crawl, Snapshot, ArchiveResult)
+    2. Spawns worker processes when there is work to do
+    3. Monitors worker health and restarts failed workers
+    4. Exits when all queues are empty (unless --daemon)
+    
+    Args:
+        daemon: Run forever (don't exit when idle)
+        watch: Just watch the queues without spawning workers (for debugging)
+    
+    Exit codes:
+        0: All work completed successfully
+        1: Error occurred
+    """
+    from archivebox.workers.orchestrator import Orchestrator
+    
+    if Orchestrator.is_running():
+        print('[yellow]Orchestrator is already running[/yellow]')
+        return 0
+    
+    try:
+        orchestrator_instance = Orchestrator(exit_on_idle=not daemon)
+        orchestrator_instance.runloop()
+        return 0
+    except KeyboardInterrupt:
+        return 0
+    except Exception as e:
+        print(f'[red]Orchestrator error: {type(e).__name__}: {e}[/red]', file=sys.stderr)
+        return 1
+
+
+@click.command()
+@click.option('--daemon', '-d', is_flag=True, help="Run forever (don't exit on idle)")
+@click.option('--watch', '-w', is_flag=True, help="Watch queues without spawning workers")
+@docstring(orchestrator.__doc__)
+def main(daemon: bool, watch: bool):
+    """Start the ArchiveBox orchestrator process"""
+    sys.exit(orchestrator(daemon=daemon, watch=watch))
+
+
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_pluginmap.py

@@ -0,0 +1,356 @@
+#!/usr/bin/env python3
+
+__package__ = 'archivebox.cli'
+
+from typing import Optional
+from pathlib import Path
+
+import rich_click as click
+
+from archivebox.misc.util import docstring, enforce_types
+
+
+# State Machine ASCII Art Diagrams
+CRAWL_MACHINE_DIAGRAM = """
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                              CrawlMachine                                   │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                             │
+│   ┌─────────────┐                                                           │
+│   │   QUEUED    │◄────────────────┐                                         │
+│   │  (initial)  │                 │                                         │
+│   └──────┬──────┘                 │                                         │
+│          │                        │ tick() unless can_start()               │
+│          │ tick() when            │                                         │
+│          │ can_start()            │                                         │
+│          ▼                        │                                         │
+│   ┌─────────────┐                 │                                         │
+│   │   STARTED   │─────────────────┘                                         │
+│   │             │◄────────────────┐                                         │
+│   │ enter:      │                 │                                         │
+│   │  crawl.run()│                 │ tick() unless is_finished()             │
+│   │  (discover  │                 │                                         │
+│   │   Crawl     │─────────────────┘                                         │
+│   │   hooks)    │                                                           │
+│   └──────┬──────┘                                                           │
+│          │                                                                  │
+│          │ tick() when is_finished()                                        │
+│          ▼                                                                  │
+│   ┌─────────────┐                                                           │
+│   │   SEALED    │                                                           │
+│   │   (final)   │                                                           │
+│   │             │                                                           │
+│   │ enter:      │                                                           │
+│   │  cleanup()  │                                                           │
+│   └─────────────┘                                                           │
+│                                                                             │
+│   Hooks triggered: on_Crawl__* (during STARTED.enter via crawl.run())       │
+│                    on_CrawlEnd__* (during SEALED.enter via cleanup())       │
+└─────────────────────────────────────────────────────────────────────────────┘
+"""
+
+SNAPSHOT_MACHINE_DIAGRAM = """
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                            SnapshotMachine                                  │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                             │
+│   ┌─────────────┐                                                           │
+│   │   QUEUED    │◄────────────────┐                                         │
+│   │  (initial)  │                 │                                         │
+│   └──────┬──────┘                 │                                         │
+│          │                        │ tick() unless can_start()               │
+│          │ tick() when            │                                         │
+│          │ can_start()            │                                         │
+│          ▼                        │                                         │
+│   ┌─────────────┐                 │                                         │
+│   │   STARTED   │─────────────────┘                                         │
+│   │             │◄────────────────┐                                         │
+│   │ enter:      │                 │                                         │
+│   │ snapshot    │                 │ tick() unless is_finished()             │
+│   │  .run()     │                 │                                         │
+│   │ (discover   │─────────────────┘                                         │
+│   │  Snapshot   │                                                           │
+│   │  hooks,     │                                                           │
+│   │  create     │                                                           │
+│   │  pending    │                                                           │
+│   │  results)   │                                                           │
+│   └──────┬──────┘                                                           │
+│          │                                                                  │
+│          │ tick() when is_finished()                                        │
+│          ▼                                                                  │
+│   ┌─────────────┐                                                           │
+│   │   SEALED    │                                                           │
+│   │   (final)   │                                                           │
+│   │             │                                                           │
+│   │ enter:      │                                                           │
+│   │  cleanup()  │                                                           │
+│   └─────────────┘                                                           │
+│                                                                             │
+│   Hooks triggered: on_Snapshot__* (creates ArchiveResults in STARTED.enter) │
+└─────────────────────────────────────────────────────────────────────────────┘
+"""
+
+ARCHIVERESULT_MACHINE_DIAGRAM = """
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                          ArchiveResultMachine                               │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                             │
+│   ┌─────────────┐                                                           │
+│   │   QUEUED    │◄────────────────┐                                         │
+│   │  (initial)  │                 │                                         │
+│   └──────┬──────┘                 │                                         │
+│          │                        │ tick() unless can_start()               │
+│          │ tick() when            │                                         │
+│          │ can_start()            │                                         │
+│          ▼                        │                                         │
+│   ┌─────────────┐                 │                                         │
+│   │   STARTED   │─────────────────┘                                         │
+│   │             │◄────────────────┐                                         │
+│   │ enter:      │                 │ tick() unless is_finished()             │
+│   │ result.run()│─────────────────┘                                         │
+│   │ (execute    │                                                           │
+│   │  hook via   │                                                           │
+│   │  run_hook())│                                                           │
+│   └──────┬──────┘                                                           │
+│          │                                                                  │
+│          │ tick() checks status set by hook output                          │
+│          ├────────────────┬────────────────┬────────────────┐               │
+│          ▼                ▼                ▼                ▼               │
+│   ┌───────────┐    ┌───────────┐    ┌───────────┐    ┌───────────┐          │
+│   │ SUCCEEDED │    │  FAILED   │    │  SKIPPED  │    │  BACKOFF  │          │
+│   │  (final)  │    │  (final)  │    │  (final)  │    │           │          │
+│   └───────────┘    └───────────┘    └───────────┘    └─────┬─────┘          │
+│                                                            │                │
+│                                              can_start()───┘                │
+│                                              loops back to STARTED          │
+│                                                                             │
+│   Each ArchiveResult runs ONE specific hook (stored in .hook_name field)    │
+└─────────────────────────────────────────────────────────────────────────────┘
+"""
+
+BINARY_MACHINE_DIAGRAM = """
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                             BinaryMachine                                   │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                             │
+│   ┌─────────────┐                                                           │
+│   │   QUEUED    │◄────────────────┐                                         │
+│   │  (initial)  │                 │                                         │
+│   └──────┬──────┘                 │                                         │
+│          │                        │ tick() unless can_start()               │
+│          │ tick() when            │                                         │
+│          │ can_start()            │                                         │
+│          ▼                        │                                         │
+│   ┌─────────────┐                 │                                         │
+│   │   STARTED   │─────────────────┘                                         │
+│   │             │◄────────────────┐                                         │
+│   │ enter:      │                 │                                         │
+│   │ binary.run()│                 │ tick() unless is_finished()             │
+│   │ (discover   │─────────────────┘                                         │
+│   │  Binary     │                                                           │
+│   │  hooks,     │                                                           │
+│   │  try each   │                                                           │
+│   │  provider)  │                                                           │
+│   └──────┬──────┘                                                           │
+│          │                                                                  │
+│          │ tick() checks status set by hook output                          │
+│          ├────────────────────────────────┐                                 │
+│          ▼                                ▼                                 │
+│   ┌─────────────┐                  ┌─────────────┐                          │
+│   │  SUCCEEDED  │                  │   FAILED    │                          │
+│   │   (final)   │                  │   (final)   │                          │
+│   │             │                  │             │                          │
+│   │ abspath,    │                  │ no provider │                          │
+│   │ version set │                  │ succeeded   │                          │
+│   └─────────────┘                  └─────────────┘                          │
+│                                                                             │
+│   Hooks triggered: on_Binary__* (provider hooks during STARTED.enter)       │
+│   Providers tried in sequence until one succeeds: apt, brew, pip, npm, etc. │
+└─────────────────────────────────────────────────────────────────────────────┘
+"""
+
+
+@enforce_types
+def pluginmap(
+    show_disabled: bool = False,
+    model: Optional[str] = None,
+    quiet: bool = False,
+) -> dict:
+    """
+    Show a map of all state machines and their associated plugin hooks.
+
+    Displays ASCII art diagrams of the core model state machines (Crawl, Snapshot,
+    ArchiveResult, Binary) and lists all auto-detected on_Modelname_xyz hooks
+    that will run for each model's transitions.
+    """
+    from rich.console import Console
+    from rich.table import Table
+    from rich.panel import Panel
+    from rich import box
+
+    from archivebox.hooks import (
+        discover_hooks,
+        extract_step,
+        is_background_hook,
+        BUILTIN_PLUGINS_DIR,
+        USER_PLUGINS_DIR,
+    )
+
+    console = Console()
+    prnt = console.print
+
+    # Model event types that can have hooks
+    model_events = {
+        'Crawl': {
+            'description': 'Hooks run when a Crawl starts (QUEUED→STARTED)',
+            'machine': 'CrawlMachine',
+            'diagram': CRAWL_MACHINE_DIAGRAM,
+        },
+        'CrawlEnd': {
+            'description': 'Hooks run when a Crawl finishes (STARTED→SEALED)',
+            'machine': 'CrawlMachine',
+            'diagram': None,  # Part of CrawlMachine
+        },
+        'Snapshot': {
+            'description': 'Hooks run for each Snapshot (creates ArchiveResults)',
+            'machine': 'SnapshotMachine',
+            'diagram': SNAPSHOT_MACHINE_DIAGRAM,
+        },
+        'Binary': {
+            'description': 'Hooks for installing binary dependencies (providers)',
+            'machine': 'BinaryMachine',
+            'diagram': BINARY_MACHINE_DIAGRAM,
+        },
+    }
+
+    # Filter to specific model if requested
+    if model:
+        model = model.title()
+        if model not in model_events:
+            prnt(f'[red]Error: Unknown model "{model}". Available: {", ".join(model_events.keys())}[/red]')
+            return {}
+        model_events = {model: model_events[model]}
+
+    result = {
+        'models': {},
+        'plugins_dir': str(BUILTIN_PLUGINS_DIR),
+        'user_plugins_dir': str(USER_PLUGINS_DIR),
+    }
+
+    if not quiet:
+        prnt()
+        prnt('[bold cyan]ArchiveBox Plugin Map[/bold cyan]')
+        prnt(f'[dim]Built-in plugins: {BUILTIN_PLUGINS_DIR}[/dim]')
+        prnt(f'[dim]User plugins: {USER_PLUGINS_DIR}[/dim]')
+        prnt()
+
+    # Show diagrams first (unless quiet mode)
+    if not quiet:
+        # Show ArchiveResult diagram separately since it's different
+        prnt(Panel(
+            ARCHIVERESULT_MACHINE_DIAGRAM,
+            title='[bold green]ArchiveResultMachine[/bold green]',
+            border_style='green',
+            expand=False,
+        ))
+        prnt()
+
+    for event_name, info in model_events.items():
+        # Discover hooks for this event
+        hooks = discover_hooks(event_name, filter_disabled=not show_disabled)
+
+        # Build hook info list
+        hook_infos = []
+        for hook_path in hooks:
+            # Get plugin name from parent directory (e.g., 'wget' from 'plugins/wget/on_Snapshot__61_wget.py')
+            plugin_name = hook_path.parent.name
+            step = extract_step(hook_path.name)
+            is_bg = is_background_hook(hook_path.name)
+
+            hook_infos.append({
+                'path': str(hook_path),
+                'name': hook_path.name,
+                'plugin': plugin_name,
+                'step': step,
+                'is_background': is_bg,
+                'extension': hook_path.suffix,
+            })
+
+        result['models'][event_name] = {
+            'description': info['description'],
+            'machine': info['machine'],
+            'hooks': hook_infos,
+            'hook_count': len(hook_infos),
+        }
+
+        if not quiet:
+            # Show diagram if this model has one
+            if info.get('diagram'):
+                prnt(Panel(
+                    info['diagram'],
+                    title=f'[bold green]{info["machine"]}[/bold green]',
+                    border_style='green',
+                    expand=False,
+                ))
+                prnt()
+
+            # Create hooks table
+            table = Table(
+                title=f'[bold yellow]on_{event_name}__* Hooks[/bold yellow] ({len(hooks)} found)',
+                box=box.ROUNDED,
+                show_header=True,
+                header_style='bold magenta',
+            )
+            table.add_column('Step', justify='center', width=6)
+            table.add_column('Plugin', style='cyan', width=20)
+            table.add_column('Hook Name', style='green')
+            table.add_column('BG', justify='center', width=4)
+            table.add_column('Type', justify='center', width=5)
+
+            # Sort by step then by name
+            sorted_hooks = sorted(hook_infos, key=lambda h: (h['step'], h['name']))
+
+            for hook in sorted_hooks:
+                bg_marker = '[yellow]bg[/yellow]' if hook['is_background'] else ''
+                ext = hook['extension'].lstrip('.')
+                table.add_row(
+                    str(hook['step']),
+                    hook['plugin'],
+                    hook['name'],
+                    bg_marker,
+                    ext,
+                )
+
+            prnt(table)
+            prnt()
+            prnt(f'[dim]{info["description"]}[/dim]')
+            prnt()
+
+    # Summary
+    if not quiet:
+        total_hooks = sum(m['hook_count'] for m in result['models'].values())
+        prnt(f'[bold]Total hooks discovered: {total_hooks}[/bold]')
+        prnt()
+        prnt('[dim]Hook naming convention: on_{Model}__{XX}_{description}[.bg].{ext}[/dim]')
+        prnt('[dim]  - XX: Two-digit order (first digit = step 0-9)[/dim]')
+        prnt('[dim]  - .bg: Background hook (non-blocking)[/dim]')
+        prnt('[dim]  - ext: py, sh, or js[/dim]')
+        prnt()
+
+    return result
+
+
+@click.command()
+@click.option('--show-disabled', '-a', is_flag=True, help='Show hooks from disabled plugins too')
+@click.option('--model', '-m', type=str, default=None, help='Filter to specific model (Crawl, Snapshot, Binary, CrawlEnd)')
+@click.option('--quiet', '-q', is_flag=True, help='Output JSON only, no ASCII diagrams')
+@docstring(pluginmap.__doc__)
+def main(**kwargs):
+    import json
+    result = pluginmap(**kwargs)
+    if kwargs.get('quiet'):
+        print(json.dumps(result, indent=2))
+
+
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_process.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+"""
+archivebox process <action> [--filters]
+
+Manage Process records (system-managed, mostly read-only).
+
+Process records track executions of binaries during extraction.
+They are created automatically by the system and are primarily for debugging.
+
+Actions:
+    list    - List Processes as JSONL (with optional filters)
+
+Examples:
+    # List all processes
+    archivebox process list
+
+    # List processes by binary
+    archivebox process list --binary-name=chrome
+
+    # List recent processes
+    archivebox process list --limit=10
+"""
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox process'
+
+import sys
+from typing import Optional
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+def list_processes(
+    binary_name: Optional[str] = None,
+    machine_id: Optional[str] = None,
+    limit: Optional[int] = None,
+) -> int:
+    """
+    List Processes as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.machine.models import Process
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Process.objects.all().select_related('binary', 'machine').order_by('-start_ts')
+
+    # Apply filters
+    filter_kwargs = {}
+    if binary_name:
+        filter_kwargs['binary__name'] = binary_name
+    if machine_id:
+        filter_kwargs['machine_id'] = machine_id
+
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for process in queryset:
+        if is_tty:
+            binary_name_str = process.binary.name if process.binary else 'unknown'
+            exit_code = process.returncode if process.returncode is not None else '?'
+            status_color = 'green' if process.returncode == 0 else 'red' if process.returncode else 'yellow'
+            rprint(f'[{status_color}]exit={exit_code:3}[/{status_color}] [cyan]{binary_name_str:15}[/cyan] [dim]{process.id}[/dim]')
+        else:
+            write_record(process.to_json())
+        count += 1
+
+    rprint(f'[dim]Listed {count} processes[/dim]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+@click.group()
+def main():
+    """Manage Process records (read-only, system-managed)."""
+    pass
+
+
+@main.command('list')
+@click.option('--binary-name', '-b', help='Filter by binary name')
+@click.option('--machine-id', '-m', help='Filter by machine ID')
+@click.option('--limit', '-n', type=int, help='Limit number of results')
+def list_cmd(binary_name: Optional[str], machine_id: Optional[str], limit: Optional[int]):
+    """List Processes as JSONL."""
+    sys.exit(list_processes(
+        binary_name=binary_name,
+        machine_id=machine_id,
+        limit=limit,
+    ))
+
+
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_remove.py

@@ -3,80 +3,96 @@
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox remove'
 
-import sys
-import argparse
+import shutil
+from pathlib import Path
+from typing import Iterable
 
-from typing import Optional, List, IO
+import rich_click as click
 
-from ..main import remove
-from ..util import docstring
-from ..config import OUTPUT_DIR
-from ..logging_util import SmartFormatter, accept_stdin
+from django.db.models import QuerySet
+
+from archivebox.config import DATA_DIR
+from archivebox.config.django import setup_django
+from archivebox.misc.util import enforce_types, docstring
+from archivebox.misc.checks import check_data_folder
+from archivebox.misc.logging_util import (
+    log_list_started,
+    log_list_finished,
+    log_removal_started,
+    log_removal_finished,
+    TimedProgress,
+)
 
 
+@enforce_types
+def remove(filter_patterns: Iterable[str]=(),
+          filter_type: str='exact',
+          snapshots: QuerySet | None=None,
+          after: float | None=None,
+          before: float | None=None,
+          yes: bool=False,
+          delete: bool=False,
+          out_dir: Path=DATA_DIR) -> QuerySet:
+    """Remove the specified URLs from the archive"""
+    
+    setup_django()
+    check_data_folder()
+    
+    from archivebox.cli.archivebox_search import get_snapshots
+
+    log_list_started(filter_patterns, filter_type)
+    timer = TimedProgress(360, prefix='      ')
+    try:
+        snapshots = get_snapshots(
+            snapshots=snapshots,
+            filter_patterns=list(filter_patterns) if filter_patterns else None,
+            filter_type=filter_type,
+            after=after,
+            before=before,
+        )
+    finally:
+        timer.end()
+
+    if not snapshots.exists():
+        log_removal_finished(0, 0)
+        raise SystemExit(1)
+
+    log_list_finished(snapshots)
+    log_removal_started(snapshots, yes=yes, delete=delete)
+
+    timer = TimedProgress(360, prefix='      ')
+    try:
+        for snapshot in snapshots:
+            if delete:
+                shutil.rmtree(snapshot.output_dir, ignore_errors=True)
+    finally:
+        timer.end()
+
+    to_remove = snapshots.count()
+
+    from archivebox.search import flush_search_index
+    from archivebox.core.models import Snapshot
+
+    flush_search_index(snapshots=snapshots)
+    snapshots.delete()
+    all_snapshots = Snapshot.objects.all()
+    log_removal_finished(all_snapshots.count(), to_remove)
+
+    return all_snapshots
+
+
+@click.command()
+@click.option('--yes', is_flag=True, help='Remove links instantly without prompting to confirm')
+@click.option('--delete', is_flag=True, help='Delete the archived content and metadata folder in addition to removing from index')
+@click.option('--before', type=float, help='Remove only URLs bookmarked before timestamp')
+@click.option('--after', type=float, help='Remove only URLs bookmarked after timestamp')
+@click.option('--filter-type', '-f', type=click.Choice(('exact', 'substring', 'domain', 'regex', 'tag')), default='exact', help='Type of pattern matching to use when filtering URLs')
+@click.argument('filter_patterns', nargs=-1)
 @docstring(remove.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=remove.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        '--yes', # '-y',
-        action='store_true',
-        help='Remove links instantly without prompting to confirm.',
-    )
-    parser.add_argument(
-        '--delete', # '-r',
-        action='store_true',
-        help=(
-            "In addition to removing the link from the index, "
-            "also delete its archived content and metadata folder."
-        ),
-    )
-    parser.add_argument(
-        '--before', #'-b',
-        type=float,
-        help="List only URLs bookmarked before the given timestamp.",
-        default=None,
-    )
-    parser.add_argument(
-        '--after', #'-a',
-        type=float,
-        help="List only URLs bookmarked after the given timestamp.",
-        default=None,
-    )
-    parser.add_argument(
-        '--filter-type',
-        type=str,
-        choices=('exact', 'substring', 'domain', 'regex','tag'),
-        default='exact',
-        help='Type of pattern matching to use when filtering URLs',
-    )
-    parser.add_argument(
-        'filter_patterns',
-        nargs='*',
-        type=str,
-        help='URLs matching this filter pattern will be removed from the index.'
-    )
-    command = parser.parse_args(args or ())
-    
-    filter_str = None
-    if not command.filter_patterns:
-        filter_str = accept_stdin(stdin)
+def main(**kwargs):
+    """Remove the specified URLs from the archive"""
+    remove(**kwargs)
 
-    remove(
-        filter_str=filter_str,
-        filter_patterns=command.filter_patterns,
-        filter_type=command.filter_type,
-        before=command.before,
-        after=command.after,
-        yes=command.yes,
-        delete=command.delete,
-        out_dir=pwd or OUTPUT_DIR,
-    )
-    
 
 if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main()

archivebox/cli/archivebox_run.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+
+"""
+archivebox run [--daemon]
+
+Unified command for processing queued work.
+
+Modes:
+    - With stdin JSONL: Process piped records, exit when complete
+    - Without stdin (TTY): Run orchestrator in foreground until killed
+
+Examples:
+    # Run orchestrator in foreground (replaces `archivebox orchestrator`)
+    archivebox run
+
+    # Run as daemon (don't exit on idle)
+    archivebox run --daemon
+
+    # Process specific records (pipe any JSONL type, exits when done)
+    archivebox snapshot list --status=queued | archivebox run
+    archivebox archiveresult list --status=failed | archivebox run
+    archivebox crawl list --status=queued | archivebox run
+
+    # Mixed types work too
+    cat mixed_records.jsonl | archivebox run
+"""
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox run'
+
+import sys
+
+import rich_click as click
+from rich import print as rprint
+
+
+def process_stdin_records() -> int:
+    """
+    Process JSONL records from stdin.
+
+    Create-or-update behavior:
+    - Records WITHOUT id: Create via Model.from_json(), then queue
+    - Records WITH id: Lookup existing, re-queue for processing
+
+    Outputs JSONL of all processed records (for chaining).
+
+    Handles any record type: Crawl, Snapshot, ArchiveResult.
+    Auto-cascades: Crawl → Snapshots → ArchiveResults.
+
+    Uses inline mode for fast processing (no subprocess overhead).
+
+    Returns exit code (0 = success, 1 = error).
+    """
+    from django.utils import timezone
+
+    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_CRAWL, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.core.models import Snapshot, ArchiveResult
+    from archivebox.crawls.models import Crawl
+    from archivebox.workers.orchestrator import Orchestrator
+
+    records = list(read_stdin())
+    is_tty = sys.stdout.isatty()
+
+    if not records:
+        return 0  # Nothing to process
+
+    created_by_id = get_or_create_system_user_pk()
+    queued_count = 0
+    output_records = []
+
+    for record in records:
+        record_type = record.get('type', '')
+        record_id = record.get('id')
+
+        try:
+            if record_type == TYPE_CRAWL:
+                if record_id:
+                    # Existing crawl - re-queue
+                    try:
+                        crawl = Crawl.objects.get(id=record_id)
+                    except Crawl.DoesNotExist:
+                        crawl = Crawl.from_json(record, overrides={'created_by_id': created_by_id})
+                else:
+                    # New crawl - create it
+                    crawl = Crawl.from_json(record, overrides={'created_by_id': created_by_id})
+
+                if crawl:
+                    crawl.retry_at = timezone.now()
+                    if crawl.status not in [Crawl.StatusChoices.SEALED]:
+                        crawl.status = Crawl.StatusChoices.QUEUED
+                    crawl.save()
+                    output_records.append(crawl.to_jsonl())
+                    queued_count += 1
+
+            elif record_type == TYPE_SNAPSHOT or (record.get('url') and not record_type):
+                if record_id:
+                    # Existing snapshot - re-queue
+                    try:
+                        snapshot = Snapshot.objects.get(id=record_id)
+                    except Snapshot.DoesNotExist:
+                        snapshot = Snapshot.from_json(record, overrides={'created_by_id': created_by_id})
+                else:
+                    # New snapshot - create it
+                    snapshot = Snapshot.from_json(record, overrides={'created_by_id': created_by_id})
+
+                if snapshot:
+                    snapshot.retry_at = timezone.now()
+                    if snapshot.status not in [Snapshot.StatusChoices.SEALED]:
+                        snapshot.status = Snapshot.StatusChoices.QUEUED
+                    snapshot.save()
+                    output_records.append(snapshot.to_jsonl())
+                    queued_count += 1
+
+            elif record_type == TYPE_ARCHIVERESULT:
+                if record_id:
+                    # Existing archiveresult - re-queue
+                    try:
+                        archiveresult = ArchiveResult.objects.get(id=record_id)
+                    except ArchiveResult.DoesNotExist:
+                        archiveresult = ArchiveResult.from_json(record)
+                else:
+                    # New archiveresult - create it
+                    archiveresult = ArchiveResult.from_json(record)
+
+                if archiveresult:
+                    archiveresult.retry_at = timezone.now()
+                    if archiveresult.status in [ArchiveResult.StatusChoices.FAILED, ArchiveResult.StatusChoices.SKIPPED, ArchiveResult.StatusChoices.BACKOFF]:
+                        archiveresult.status = ArchiveResult.StatusChoices.QUEUED
+                    archiveresult.save()
+                    output_records.append(archiveresult.to_jsonl())
+                    queued_count += 1
+
+            else:
+                # Unknown type - pass through
+                output_records.append(record)
+
+        except Exception as e:
+            rprint(f'[yellow]Error processing record: {e}[/yellow]', file=sys.stderr)
+            continue
+
+    # Output all processed records (for chaining)
+    if not is_tty:
+        for rec in output_records:
+            write_record(rec)
+
+    if queued_count == 0:
+        rprint('[yellow]No records to process[/yellow]', file=sys.stderr)
+        return 0
+
+    rprint(f'[blue]Processing {queued_count} records...[/blue]', file=sys.stderr)
+
+    # Run orchestrator with inline=True for fast processing (no subprocess overhead)
+    orchestrator = Orchestrator(exit_on_idle=True, inline=True)
+    orchestrator.runloop()
+
+    return 0
+
+
+def run_orchestrator(daemon: bool = False) -> int:
+    """
+    Run the orchestrator process.
+
+    The orchestrator:
+    1. Polls each model queue (Crawl, Snapshot, ArchiveResult)
+    2. Spawns worker processes when there is work to do
+    3. Monitors worker health and restarts failed workers
+    4. Exits when all queues are empty (unless --daemon)
+
+    Args:
+        daemon: Run forever (don't exit when idle)
+
+    Returns exit code (0 = success, 1 = error).
+    """
+    from archivebox.workers.orchestrator import Orchestrator
+
+    if Orchestrator.is_running():
+        rprint('[yellow]Orchestrator is already running[/yellow]', file=sys.stderr)
+        return 0
+
+    try:
+        orchestrator = Orchestrator(exit_on_idle=not daemon)
+        orchestrator.runloop()
+        return 0
+    except KeyboardInterrupt:
+        return 0
+    except Exception as e:
+        rprint(f'[red]Orchestrator error: {type(e).__name__}: {e}[/red]', file=sys.stderr)
+        return 1
+
+
+@click.command()
+@click.option('--daemon', '-d', is_flag=True, help="Run forever (don't exit on idle)")
+def main(daemon: bool):
+    """
+    Process queued work.
+
+    When stdin is piped: Process those specific records and exit (uses inline mode).
+    When run standalone: Run orchestrator in foreground.
+    """
+    # Check if stdin has data (non-TTY means piped input)
+    if not sys.stdin.isatty():
+        sys.exit(process_stdin_records())
+    else:
+        sys.exit(run_orchestrator(daemon=daemon))
+
+
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_schedule.py

@@ -1,116 +1,169 @@
 #!/usr/bin/env python3
 
 __package__ = 'archivebox.cli'
-__command__ = 'archivebox schedule'
 
 import sys
-import argparse
+from pathlib import Path
 
-from typing import Optional, List, IO
+import rich_click as click
+from rich import print
 
-from ..main import schedule
-from ..util import docstring
-from ..config import OUTPUT_DIR
-from ..logging_util import SmartFormatter, reject_stdin
+from archivebox.misc.util import enforce_types, docstring
+from archivebox.config import DATA_DIR, CONSTANTS
+from archivebox.config.common import ARCHIVING_CONFIG
+from archivebox.config.permissions import USER
 
 
+CRON_COMMENT = 'ArchiveBox'
+
+
+@enforce_types
+def schedule(add: bool=False,
+            show: bool=False,
+            clear: bool=False,
+            foreground: bool=False,
+            run_all: bool=False,
+            quiet: bool=False,
+            every: str | None=None,
+            tag: str='',
+            depth: int | str=0,
+            overwrite: bool=False,
+            update: bool=not ARCHIVING_CONFIG.ONLY_NEW,
+            import_path: str | None=None,
+            out_dir: Path=DATA_DIR) -> None:
+    """Set ArchiveBox to regularly import URLs at specific times using cron"""
+ 
+    depth = int(depth)
+    
+    import shutil
+    from crontab import CronTab, CronSlices
+    from archivebox.misc.system import dedupe_cron_jobs
+    
+    # Find the archivebox binary path
+    ARCHIVEBOX_ABSPATH = shutil.which('archivebox') or sys.executable.replace('python', 'archivebox')
+
+    Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
+
+    cron = CronTab(user=True)
+    cron = dedupe_cron_jobs(cron)
+
+    if clear:
+        print(cron.remove_all(comment=CRON_COMMENT))
+        cron.write()
+        raise SystemExit(0)
+
+    existing_jobs = list(cron.find_comment(CRON_COMMENT))
+
+    if every or add:
+        every = every or 'day'
+        quoted = lambda s: f'"{s}"' if (s and ' ' in str(s)) else str(s)
+        cmd = [
+            'cd',
+            quoted(out_dir),
+            '&&',
+            quoted(ARCHIVEBOX_ABSPATH),
+            *([
+                'add',
+                *(['--overwrite'] if overwrite else []),
+                *(['--update'] if update else []),
+                *([f'--tag={tag}'] if tag else []),
+                f'--depth={depth}',
+                f'"{import_path}"',
+            ] if import_path else ['update']),
+            '>>',
+            quoted(Path(CONSTANTS.LOGS_DIR) / 'schedule.log'),
+            '2>&1',
+        ]
+        new_job = cron.new(command=' '.join(cmd), comment=CRON_COMMENT)
+
+        if every in ('minute', 'hour', 'day', 'month', 'year'):
+            set_every = getattr(new_job.every(), every)
+            set_every()
+        elif CronSlices.is_valid(every):
+            new_job.setall(every)
+        else:
+            print('[red]\\[X] Got invalid timeperiod for cron task.[/red]')
+            print('    It must be one of minute/hour/day/month')
+            print('    or a quoted cron-format schedule like:')
+            print('        archivebox init --every=day --depth=1 https://example.com/some/rss/feed.xml')
+            print('        archivebox init --every="0/5 * * * *" --depth=1 https://example.com/some/rss/feed.xml')
+            raise SystemExit(1)
+
+        cron = dedupe_cron_jobs(cron)
+        print(cron)
+        cron.write()
+
+        total_runs = sum(j.frequency_per_year() for j in cron)
+        existing_jobs = list(cron.find_command('archivebox'))
+
+        print()
+        print('[green]\\[√] Scheduled new ArchiveBox cron job for user: {} ({} jobs are active).[/green]'.format(USER, len(existing_jobs)))
+        print('\n'.join(f'  > {cmd}' if str(cmd) == str(new_job) else f'    {cmd}' for cmd in existing_jobs))
+        if total_runs > 60 and not quiet:
+            print()
+            print('[yellow]\\[!] With the current cron config, ArchiveBox is estimated to run >{} times per year.[/yellow]'.format(total_runs))
+            print('    Congrats on being an enthusiastic internet archiver! 👌')
+            print()
+            print('    [violet]Make sure you have enough storage space available to hold all the data.[/violet]')
+            print('    Using a compressed/deduped filesystem like ZFS is recommended if you plan on archiving a lot.')
+            print()
+    elif show:
+        if existing_jobs:
+            print('\n'.join(str(cmd) for cmd in existing_jobs))
+        else:
+            print('[red]\\[X] There are no ArchiveBox cron jobs scheduled for your user ({}).[/red]'.format(USER))
+            print('    To schedule a new job, run:')
+            print('        archivebox schedule --every=[timeperiod] --depth=1 https://example.com/some/rss/feed.xml')
+        raise SystemExit(0)
+
+    if foreground or run_all:
+        if not existing_jobs:
+            print('[red]\\[X] You must schedule some jobs first before running in foreground mode.[/red]')
+            print('    archivebox schedule --every=hour --depth=1 https://example.com/some/rss/feed.xml')
+            raise SystemExit(1)
+
+        print('[green]\\[*] Running {} ArchiveBox jobs in foreground task scheduler...[/green]'.format(len(existing_jobs)))
+        if run_all:
+            try:
+                for job in existing_jobs:
+                    sys.stdout.write(f'  > {job.command.split("/archivebox ")[0].split(" && ")[0]}\n')
+                    sys.stdout.write(f'    > {job.command.split("/archivebox ")[-1].split(" >> ")[0]}')
+                    sys.stdout.flush()
+                    job.run()
+                    sys.stdout.write(f'\r    √ {job.command.split("/archivebox ")[-1]}\n')
+            except KeyboardInterrupt:
+                print('\n[green]\\[√] Stopped.[/green] (Ctrl+C)')
+                raise SystemExit(1)
+
+        if foreground:
+            try:
+                for job in existing_jobs:
+                    print(f'  > {job.command.split("/archivebox ")[-1].split(" >> ")[0]}')
+                for result in cron.run_scheduler():
+                    print(result)
+            except KeyboardInterrupt:
+                print('\n[green]\\[√] Stopped.[/green] (Ctrl+C)')
+                raise SystemExit(1)
+
+
+@click.command()
+@click.option('--quiet', '-q', is_flag=True, help="Don't warn about storage space")
+@click.option('--add', is_flag=True, help='Add a new scheduled ArchiveBox update job to cron')
+@click.option('--every', type=str, help='Run ArchiveBox once every [timeperiod] (hour/day/month/year or cron format e.g. "0 0 * * *")')
+@click.option('--tag', '-t', default='', help='Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3')
+@click.option('--depth', type=click.Choice(['0', '1']), default='0', help='Depth to archive to [0] or 1')
+@click.option('--overwrite', is_flag=True, help='Re-archive any URLs that have been previously archived, overwriting existing Snapshots')
+@click.option('--update', is_flag=True, help='Re-pull any URLs that have been previously added, as needed to fill missing ArchiveResults')
+@click.option('--clear', is_flag=True, help='Stop all ArchiveBox scheduled runs (remove cron jobs)')
+@click.option('--show', is_flag=True, help='Print a list of currently active ArchiveBox cron jobs')
+@click.option('--foreground', '-f', is_flag=True, help='Launch ArchiveBox scheduler as a long-running foreground task instead of using cron')
+@click.option('--run-all', is_flag=True, help='Run all the scheduled jobs once immediately, independent of their configured schedules')
+@click.argument('import_path', required=False)
 @docstring(schedule.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=schedule.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        '--quiet', '-q',
-        action='store_true',
-        help=("Don't warn about storage space."),
-    )
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        '--add', # '-a',
-        action='store_true',
-        help='Add a new scheduled ArchiveBox update job to cron',
-    )
-    parser.add_argument(
-        '--every', # '-e',
-        type=str,
-        default=None,
-        help='Run ArchiveBox once every [timeperiod] (hour/day/month/year or cron format e.g. "0 0 * * *")',
-    )
-    parser.add_argument(
-        '--tag', '-t',
-        type=str,
-        default='',
-        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
-    )
-    parser.add_argument(
-        '--depth', # '-d',
-        type=int,
-        choices=[0, 1],
-        default=0,
-        help='Depth to archive to [0] or 1, see "add" command help for more info',
-    )
-    parser.add_argument(
-        '--overwrite',
-        action='store_true',
-        help='Re-archive any URLs that have been previously archived, overwriting existing Snapshots',
-    )
-    parser.add_argument(
-        '--update',
-        action='store_true',
-        help='Re-pull any URLs that have been previously added, as needed to fill missing ArchiveResults',
-    )
-    group.add_argument(
-        '--clear', # '-c'
-        action='store_true',
-        help=("Stop all ArchiveBox scheduled runs (remove cron jobs)"),
-    )
-    group.add_argument(
-        '--show', # '-s'
-        action='store_true',
-        help=("Print a list of currently active ArchiveBox cron jobs"),
-    )
-    group.add_argument(
-        '--foreground', '-f',
-        action='store_true',
-        help=("Launch ArchiveBox scheduler as a long-running foreground task "
-              "instead of using cron."),
-    )
-    group.add_argument(
-        '--run-all', # '-a',
-        action='store_true',
-        help=("Run all the scheduled jobs once immediately, independent of "
-              "their configured schedules, can be used together with --foreground"),
-    )
-    parser.add_argument(
-        'import_path',
-        nargs='?',
-        type=str,
-        default=None,
-        help=("Check this path and import any new links on every run "
-              "(can be either local file or remote URL)"),
-    )
-    command = parser.parse_args(args or ())
-    reject_stdin(__command__, stdin)
-
-    schedule(
-        add=command.add,
-        show=command.show,
-        clear=command.clear,
-        foreground=command.foreground,
-        run_all=command.run_all,
-        quiet=command.quiet,
-        every=command.every,
-        tag=command.tag,
-        depth=command.depth,
-        overwrite=command.overwrite,
-        update=command.update,
-        import_path=command.import_path,
-        out_dir=pwd or OUTPUT_DIR,
-    )
+def main(**kwargs):
+    """Set ArchiveBox to regularly import URLs at specific times using cron"""
+    schedule(**kwargs)
 
 
 if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main()

archivebox/cli/archivebox_search.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox search'
+
+from pathlib import Path
+from typing import Optional, List, Any
+
+import rich_click as click
+from rich import print
+
+from django.db.models import QuerySet
+
+from archivebox.config import DATA_DIR
+from archivebox.misc.logging import stderr
+from archivebox.misc.util import enforce_types, docstring
+
+# Filter types for URL matching
+LINK_FILTERS = {
+    'exact': lambda pattern: {'url': pattern},
+    'substring': lambda pattern: {'url__icontains': pattern},
+    'regex': lambda pattern: {'url__iregex': pattern},
+    'domain': lambda pattern: {'url__istartswith': f'http://{pattern}'},
+    'tag': lambda pattern: {'tags__name': pattern},
+    'timestamp': lambda pattern: {'timestamp': pattern},
+}
+
+STATUS_CHOICES = ['indexed', 'archived', 'unarchived']
+
+
+
+def get_snapshots(snapshots: Optional[QuerySet]=None,
+                  filter_patterns: Optional[List[str]]=None,
+                  filter_type: str='substring',
+                  after: Optional[float]=None,
+                  before: Optional[float]=None,
+                  out_dir: Path=DATA_DIR) -> QuerySet:
+    """Filter and return Snapshots matching the given criteria."""
+    from archivebox.core.models import Snapshot
+
+    if snapshots:
+        result = snapshots
+    else:
+        result = Snapshot.objects.all()
+
+    if after is not None:
+        result = result.filter(timestamp__gte=after)
+    if before is not None:
+        result = result.filter(timestamp__lt=before)
+    if filter_patterns:
+        result = Snapshot.objects.filter_by_patterns(filter_patterns, filter_type)
+
+    if not result:
+        stderr('[!] No Snapshots matched your filters:', filter_patterns, f'({filter_type})', color='lightyellow')
+
+    return result
+
+
+@enforce_types
+def search(filter_patterns: list[str] | None=None,
+           filter_type: str='substring',
+           status: str='indexed',
+           before: float | None=None,
+           after: float | None=None,
+           sort: str | None=None,
+           json: bool=False,
+           html: bool=False,
+           csv: str | None=None,
+           with_headers: bool=False):
+    """List, filter, and export information about archive entries"""
+    from archivebox.core.models import Snapshot
+
+    if with_headers and not (json or html or csv):
+        stderr('[X] --with-headers requires --json, --html or --csv\n', color='red')
+        raise SystemExit(2)
+
+    # Query DB directly - no filesystem scanning
+    snapshots = get_snapshots(
+        filter_patterns=list(filter_patterns) if filter_patterns else None,
+        filter_type=filter_type,
+        before=before,
+        after=after,
+    )
+
+    # Apply status filter
+    if status == 'archived':
+        snapshots = snapshots.filter(downloaded_at__isnull=False)
+    elif status == 'unarchived':
+        snapshots = snapshots.filter(downloaded_at__isnull=True)
+    # 'indexed' = all snapshots (no filter)
+
+    if sort:
+        snapshots = snapshots.order_by(sort)
+
+    # Export to requested format
+    if json:
+        output = snapshots.to_json(with_headers=with_headers)
+    elif html:
+        output = snapshots.to_html(with_headers=with_headers)
+    elif csv:
+        output = snapshots.to_csv(cols=csv.split(','), header=with_headers)
+    else:
+        from archivebox.misc.logging_util import printable_folders
+        # Convert to dict for printable_folders
+        folders = {s.output_dir: s for s in snapshots}
+        output = printable_folders(folders, with_headers)
+
+    print(output)
+    return output
+
+
+@click.command()
+@click.option('--filter-type', '-f', type=click.Choice(['search', *LINK_FILTERS.keys()]), default='substring', help='Pattern matching type for filtering URLs')
+@click.option('--status', '-s', type=click.Choice(STATUS_CHOICES), default='indexed', help='List snapshots with the given status')
+@click.option('--before', '-b', type=float, help='List snapshots bookmarked before the given UNIX timestamp')
+@click.option('--after', '-a', type=float, help='List snapshots bookmarked after the given UNIX timestamp')
+@click.option('--sort', '-o', type=str, help='Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at')
+@click.option('--json', '-J', is_flag=True, help='Print output in JSON format')
+@click.option('--html', '-M', is_flag=True, help='Print output in HTML format (suitable for viewing statically without a server)')
+@click.option('--csv', '-C', type=str, help='Print output as CSV with the provided fields, e.g.: created_at,url,title')
+@click.option('--with-headers', '-H', is_flag=True, help='Include extra CSV/HTML headers in the output')
+@click.help_option('--help', '-h')
+@click.argument('filter_patterns', nargs=-1)
+@docstring(search.__doc__)
+def main(**kwargs):
+    return search(**kwargs)
+
+
+
+if __name__ == '__main__':
+    main()

archivebox/cli/archivebox_server.py

@@ -1,76 +1,136 @@
 #!/usr/bin/env python3
 
 __package__ = 'archivebox.cli'
-__command__ = 'archivebox server'
 
-import sys
-import argparse
+from typing import Iterable
 
-from typing import Optional, List, IO
+import rich_click as click
+from rich import print
 
-from ..main import server
-from ..util import docstring
-from ..config import OUTPUT_DIR, BIND_ADDR
-from ..logging_util import SmartFormatter, reject_stdin
+from archivebox.misc.util import docstring, enforce_types
+from archivebox.config.common import SERVER_CONFIG
 
-@docstring(server.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=server.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        'runserver_args',
-        nargs='*',
-        type=str,
-        default=[BIND_ADDR],
-        help='Arguments to pass to Django runserver'
-    )
-    parser.add_argument(
-        '--reload',
-        action='store_true',
-        help='Enable auto-reloading when code or templates change',
-    )
-    parser.add_argument(
-        '--debug',
-        action='store_true',
-        help='Enable DEBUG=True mode with more verbose errors',
-    )
-    parser.add_argument(
-        '--nothreading',
-        action='store_true',
-        help='Force runserver to run in single-threaded mode',
-    )
-    parser.add_argument(
-        '--init',
-        action='store_true',
-        help='Run a full archivebox init/upgrade before starting the server',
-    )
-    parser.add_argument(
-        '--quick-init', '-i',
-        action='store_true',
-        help='Run quick archivebox init/upgrade before starting the server',
-    )
-    parser.add_argument(
-        '--createsuperuser',
-        action='store_true',
-        help='Run archivebox manage createsuperuser before starting the server',
-    )
-    command = parser.parse_args(args or ())
-    reject_stdin(__command__, stdin)
+
+@enforce_types
+def server(runserver_args: Iterable[str]=(SERVER_CONFIG.BIND_ADDR,),
+          reload: bool=False,
+          init: bool=False,
+          debug: bool=False,
+          daemonize: bool=False,
+          nothreading: bool=False) -> None:
+    """Run the ArchiveBox HTTP server"""
+
+    runserver_args = list(runserver_args)
     
-    server(
-        runserver_args=command.runserver_args + (['--nothreading'] if command.nothreading else []),
-        reload=command.reload,
-        debug=command.debug,
-        init=command.init,
-        quick_init=command.quick_init,
-        createsuperuser=command.createsuperuser,
-        out_dir=pwd or OUTPUT_DIR,
-    )
+    if init:
+        from archivebox.cli.archivebox_init import init as archivebox_init
+        archivebox_init(quick=True)
+        print()
+
+    from archivebox.misc.checks import check_data_folder
+    check_data_folder()
+
+    from django.core.management import call_command
+    from django.contrib.auth.models import User
+    
+    from archivebox.config.common import SHELL_CONFIG
+    
+    if not User.objects.filter(is_superuser=True).exclude(username='system').exists():
+        print()
+        print('[violet]Hint:[/violet] To create an [bold]admin username & password[/bold] for the [deep_sky_blue3][underline][link=http://{host}:{port}/admin]Admin UI[/link][/underline][/deep_sky_blue3], run:')
+        print('      [green]archivebox manage createsuperuser[/green]')
+        print()
+
+    host = '127.0.0.1'
+    port = '8000'
+    
+    try:
+        host_and_port = [arg for arg in runserver_args if arg.replace('.', '').replace(':', '').isdigit()][0]
+        if ':' in host_and_port:
+            host, port = host_and_port.split(':')
+        else:
+            if '.' in host_and_port:
+                host = host_and_port
+            else:
+                port = host_and_port
+    except IndexError:
+        pass
+
+    if SHELL_CONFIG.DEBUG:
+        print('[green][+] Starting ArchiveBox webserver in DEBUG mode...[/green]')
+        print(f'    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]')
+        print(f'    [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link=http://{host}:{port}/admin]http://{host}:{port}/admin[/link][/deep_sky_blue3]')
+        print('    > Writing ArchiveBox error log to ./logs/errors.log')
+        if not reload:
+            runserver_args.append('--noreload')  # '--insecure'
+        if nothreading:
+            runserver_args.append('--nothreading')
+        call_command("runserver", *runserver_args)
+    else:
+        from archivebox.workers.supervisord_util import (
+            get_existing_supervisord_process,
+            get_worker,
+            start_server_workers,
+            tail_multiple_worker_logs,
+            is_port_in_use,
+        )
+        from archivebox.workers.orchestrator import Orchestrator
+        import sys
+
+        # Check if port is already in use
+        if is_port_in_use(host, int(port)):
+            print(f'[red][X] Error: Port {port} is already in use[/red]')
+            print(f'    Another process (possibly daphne) is already listening on {host}:{port}')
+            print(f'    Stop the conflicting process or choose a different port')
+            sys.exit(1)
+
+        # Check if orchestrator is already running for this data directory
+        if Orchestrator.is_running():
+            print(f'[red][X] Error: ArchiveBox orchestrator is already running for this data directory[/red]')
+            print(f'    Stop the existing orchestrator before starting a new server')
+            print(f'    To stop: pkill -f "archivebox manage orchestrator"')
+            sys.exit(1)
+
+        # Check if supervisord is already running
+        supervisor = get_existing_supervisord_process()
+        if supervisor:
+            daphne_proc = get_worker(supervisor, 'worker_daphne')
+
+            # If daphne is already running, error out
+            if daphne_proc and daphne_proc.get('statename') == 'RUNNING':
+                orchestrator_proc = get_worker(supervisor, 'worker_orchestrator')
+                print('[red][X] Error: ArchiveBox server is already running[/red]')
+                print(f'    [green]√[/green] Web server (worker_daphne) is RUNNING on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]')
+                if orchestrator_proc and orchestrator_proc.get('statename') == 'RUNNING':
+                    print(f'    [green]√[/green] Background worker (worker_orchestrator) is RUNNING')
+                print()
+                print('[yellow]To stop the existing server, run:[/yellow]')
+                print('    pkill -f "archivebox server"')
+                print('    pkill -f supervisord')
+                sys.exit(1)
+            # Otherwise, daphne is not running - fall through to start it
+
+        # No existing workers found - start new ones
+        print('[green][+] Starting ArchiveBox webserver...[/green]')
+        print(f'    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]')
+        print(f'    [green]>[/green] Log in to ArchiveBox Admin UI on [deep_sky_blue3][link=http://{host}:{port}/admin]http://{host}:{port}/admin[/link][/deep_sky_blue3]')
+        print('    > Writing ArchiveBox error log to ./logs/errors.log')
+        print()
+        start_server_workers(host=host, port=port, daemonize=daemonize)
+        print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
+
+
+@click.command()
+@click.argument('runserver_args', nargs=-1)
+@click.option('--reload', is_flag=True, help='Enable auto-reloading when code or templates change')
+@click.option('--debug', is_flag=True, help='Enable DEBUG=True mode with more verbose errors')
+@click.option('--nothreading', is_flag=True, help='Force runserver to run in single-threaded mode')
+@click.option('--init', is_flag=True, help='Run a full archivebox init/upgrade before starting the server')
+@click.option('--daemonize', is_flag=True, help='Run the server in the background as a daemon')
+@docstring(server.__doc__)
+def main(**kwargs):
+    server(**kwargs)
 
 
 if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main()

archivebox/cli/archivebox_setup.py

@@ -1,40 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = 'archivebox.cli'
-__command__ = 'archivebox setup'
-
-import sys
-import argparse
-
-from typing import Optional, List, IO
-
-from ..main import setup
-from ..util import docstring
-from ..config import OUTPUT_DIR
-from ..logging_util import SmartFormatter, reject_stdin
-
-
-@docstring(setup.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=setup.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    # parser.add_argument(
-    #     '--force', # '-f',
-    #     action='store_true',
-    #     help='Overwrite any existing packages that conflict with the ones ArchiveBox is trying to install',
-    # )
-    command = parser.parse_args(args or ())   # noqa
-    reject_stdin(__command__, stdin)
-
-    setup(
-        # force=command.force,
-        out_dir=pwd or OUTPUT_DIR,
-    )
-    
-
-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/cli/archivebox_shell.py

@@ -1,34 +1,27 @@
 #!/usr/bin/env python3
 
 __package__ = 'archivebox.cli'
-__command__ = 'archivebox shell'
 
-import sys
-import argparse
+from typing import Iterable
 
-from typing import Optional, List, IO
+import rich_click as click
 
-from ..main import shell
-from ..util import docstring
-from ..config import OUTPUT_DIR
-from ..logging_util import SmartFormatter, reject_stdin
+from archivebox.misc.util import docstring
 
 
+def shell(args: Iterable[str]=()) -> None:
+    """Enter an interactive ArchiveBox Django shell"""
+
+    from django.core.management import call_command
+    call_command("shell_plus", *args)
+
+
+@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
+@click.argument('args', nargs=-1)
 @docstring(shell.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=shell.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.parse_args(args or ())
-    reject_stdin(__command__, stdin)
-    
-    shell(
-        out_dir=pwd or OUTPUT_DIR,
-    )
-    
+def main(args: Iterable[str]=()) -> None:
+    shell(args=args)
+
 
 if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main()

archivebox/cli/archivebox_snapshot.py

@@ -0,0 +1,391 @@
+#!/usr/bin/env python3
+
+"""
+archivebox snapshot <action> [args...] [--filters]
+
+Manage Snapshot records.
+
+Actions:
+    create  - Create Snapshots from URLs or Crawl JSONL
+    list    - List Snapshots as JSONL (with optional filters)
+    update  - Update Snapshots from stdin JSONL
+    delete  - Delete Snapshots from stdin JSONL
+
+Examples:
+    # Create
+    archivebox snapshot create https://example.com --tag=news
+    archivebox crawl create https://example.com | archivebox snapshot create
+
+    # List with filters
+    archivebox snapshot list --status=queued
+    archivebox snapshot list --url__icontains=example.com
+
+    # Update
+    archivebox snapshot list --tag=old | archivebox snapshot update --tag=new
+
+    # Delete
+    archivebox snapshot list --url__icontains=spam.com | archivebox snapshot delete --yes
+"""
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox snapshot'
+
+import sys
+from typing import Optional, Iterable
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+# =============================================================================
+# CREATE
+# =============================================================================
+
+def create_snapshots(
+    urls: Iterable[str],
+    tag: str = '',
+    status: str = 'queued',
+    depth: int = 0,
+    created_by_id: Optional[int] = None,
+) -> int:
+    """
+    Create Snapshots from URLs or stdin JSONL (Crawl or Snapshot records).
+    Pass-through: Records that are not Crawl/Snapshot/URL are output unchanged.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from archivebox.misc.jsonl import (
+        read_args_or_stdin, write_record,
+        TYPE_SNAPSHOT, TYPE_CRAWL
+    )
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.core.models import Snapshot
+    from archivebox.crawls.models import Crawl
+
+    created_by_id = created_by_id or get_or_create_system_user_pk()
+    is_tty = sys.stdout.isatty()
+
+    # Collect all input records
+    records = list(read_args_or_stdin(urls))
+
+    if not records:
+        rprint('[yellow]No URLs or Crawls provided. Pass URLs as arguments or via stdin.[/yellow]', file=sys.stderr)
+        return 1
+
+    # Process each record - handle Crawls and plain URLs/Snapshots
+    created_snapshots = []
+    pass_through_count = 0
+
+    for record in records:
+        record_type = record.get('type', '')
+
+        try:
+            if record_type == TYPE_CRAWL:
+                # Pass through the Crawl record itself first
+                if not is_tty:
+                    write_record(record)
+
+                # Input is a Crawl - get or create it, then create Snapshots for its URLs
+                crawl = None
+                crawl_id = record.get('id')
+                if crawl_id:
+                    try:
+                        crawl = Crawl.objects.get(id=crawl_id)
+                    except Crawl.DoesNotExist:
+                        crawl = Crawl.from_json(record, overrides={'created_by_id': created_by_id})
+                else:
+                    crawl = Crawl.from_json(record, overrides={'created_by_id': created_by_id})
+
+                if not crawl:
+                    continue
+
+                # Create snapshots for each URL in the crawl
+                for url in crawl.get_urls_list():
+                    merged_tags = crawl.tags_str
+                    if tag:
+                        merged_tags = f"{merged_tags},{tag}" if merged_tags else tag
+                    snapshot_record = {
+                        'url': url,
+                        'tags': merged_tags,
+                        'crawl_id': str(crawl.id),
+                        'depth': depth,
+                        'status': status,
+                    }
+                    snapshot = Snapshot.from_json(snapshot_record, overrides={'created_by_id': created_by_id})
+                    if snapshot:
+                        created_snapshots.append(snapshot)
+                        if not is_tty:
+                            write_record(snapshot.to_jsonl())
+
+            elif record_type == TYPE_SNAPSHOT or record.get('url'):
+                # Input is a Snapshot or plain URL
+                if tag and not record.get('tags'):
+                    record['tags'] = tag
+                if status:
+                    record['status'] = status
+                record['depth'] = record.get('depth', depth)
+
+                snapshot = Snapshot.from_json(record, overrides={'created_by_id': created_by_id})
+                if snapshot:
+                    created_snapshots.append(snapshot)
+                    if not is_tty:
+                        write_record(snapshot.to_jsonl())
+
+            else:
+                # Pass-through: output records we don't handle
+                if not is_tty:
+                    write_record(record)
+                pass_through_count += 1
+
+        except Exception as e:
+            rprint(f'[red]Error creating snapshot: {e}[/red]', file=sys.stderr)
+            continue
+
+    if not created_snapshots:
+        if pass_through_count > 0:
+            rprint(f'[dim]Passed through {pass_through_count} records, no new snapshots[/dim]', file=sys.stderr)
+            return 0
+        rprint('[red]No snapshots created[/red]', file=sys.stderr)
+        return 1
+
+    rprint(f'[green]Created {len(created_snapshots)} snapshots[/green]', file=sys.stderr)
+
+    if is_tty:
+        for snapshot in created_snapshots:
+            rprint(f'  [dim]{snapshot.id}[/dim] {snapshot.url[:60]}', file=sys.stderr)
+
+    return 0
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+def list_snapshots(
+    status: Optional[str] = None,
+    url__icontains: Optional[str] = None,
+    url__istartswith: Optional[str] = None,
+    tag: Optional[str] = None,
+    crawl_id: Optional[str] = None,
+    limit: Optional[int] = None,
+) -> int:
+    """
+    List Snapshots as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.core.models import Snapshot
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = Snapshot.objects.all().order_by('-created_at')
+
+    # Apply filters
+    filter_kwargs = {
+        'status': status,
+        'url__icontains': url__icontains,
+        'url__istartswith': url__istartswith,
+        'crawl_id': crawl_id,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    # Tag filter requires special handling (M2M)
+    if tag:
+        queryset = queryset.filter(tags__name__iexact=tag)
+
+    count = 0
+    for snapshot in queryset:
+        if is_tty:
+            status_color = {
+                'queued': 'yellow',
+                'started': 'blue',
+                'sealed': 'green',
+            }.get(snapshot.status, 'dim')
+            rprint(f'[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}')
+        else:
+            write_record(snapshot.to_jsonl())
+        count += 1
+
+    rprint(f'[dim]Listed {count} snapshots[/dim]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# UPDATE
+# =============================================================================
+
+def update_snapshots(
+    status: Optional[str] = None,
+    tag: Optional[str] = None,
+) -> int:
+    """
+    Update Snapshots from stdin JSONL.
+
+    Reads Snapshot records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from django.utils import timezone
+
+    from archivebox.misc.jsonl import read_stdin, write_record
+    from archivebox.core.models import Snapshot
+
+    is_tty = sys.stdout.isatty()
+
+    records = list(read_stdin())
+    if not records:
+        rprint('[yellow]No records provided via stdin[/yellow]', file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        snapshot_id = record.get('id')
+        if not snapshot_id:
+            continue
+
+        try:
+            snapshot = Snapshot.objects.get(id=snapshot_id)
+
+            # Apply updates from CLI flags (override stdin values)
+            if status:
+                snapshot.status = status
+                snapshot.retry_at = timezone.now()
+            if tag:
+                # Add tag to existing tags
+                snapshot.save()  # Ensure saved before M2M
+                from archivebox.core.models import Tag
+                tag_obj, _ = Tag.objects.get_or_create(name=tag)
+                snapshot.tags.add(tag_obj)
+
+            snapshot.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(snapshot.to_jsonl())
+
+        except Snapshot.DoesNotExist:
+            rprint(f'[yellow]Snapshot not found: {snapshot_id}[/yellow]', file=sys.stderr)
+            continue
+
+    rprint(f'[green]Updated {updated_count} snapshots[/green]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# DELETE
+# =============================================================================
+
+def delete_snapshots(yes: bool = False, dry_run: bool = False) -> int:
+    """
+    Delete Snapshots from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
+    from archivebox.core.models import Snapshot
+
+    records = list(read_stdin())
+    if not records:
+        rprint('[yellow]No records provided via stdin[/yellow]', file=sys.stderr)
+        return 1
+
+    snapshot_ids = [r.get('id') for r in records if r.get('id')]
+
+    if not snapshot_ids:
+        rprint('[yellow]No valid snapshot IDs in input[/yellow]', file=sys.stderr)
+        return 1
+
+    snapshots = Snapshot.objects.filter(id__in=snapshot_ids)
+    count = snapshots.count()
+
+    if count == 0:
+        rprint('[yellow]No matching snapshots found[/yellow]', file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f'[yellow]Would delete {count} snapshots (dry run)[/yellow]', file=sys.stderr)
+        for snapshot in snapshots:
+            rprint(f'  [dim]{snapshot.id}[/dim] {snapshot.url[:60]}', file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint('[red]Use --yes to confirm deletion[/red]', file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = snapshots.delete()
+    rprint(f'[green]Deleted {deleted_count} snapshots[/green]', file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+@click.group()
+def main():
+    """Manage Snapshot records."""
+    pass
+
+
+@main.command('create')
+@click.argument('urls', nargs=-1)
+@click.option('--tag', '-t', default='', help='Comma-separated tags to add')
+@click.option('--status', '-s', default='queued', help='Initial status (default: queued)')
+@click.option('--depth', '-d', type=int, default=0, help='Crawl depth (default: 0)')
+def create_cmd(urls: tuple, tag: str, status: str, depth: int):
+    """Create Snapshots from URLs or stdin JSONL."""
+    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
+
+
+@main.command('list')
+@click.option('--status', '-s', help='Filter by status (queued, started, sealed)')
+@click.option('--url__icontains', help='Filter by URL contains')
+@click.option('--url__istartswith', help='Filter by URL starts with')
+@click.option('--tag', '-t', help='Filter by tag name')
+@click.option('--crawl-id', help='Filter by crawl ID')
+@click.option('--limit', '-n', type=int, help='Limit number of results')
+def list_cmd(status: Optional[str], url__icontains: Optional[str], url__istartswith: Optional[str],
+             tag: Optional[str], crawl_id: Optional[str], limit: Optional[int]):
+    """List Snapshots as JSONL."""
+    sys.exit(list_snapshots(
+        status=status,
+        url__icontains=url__icontains,
+        url__istartswith=url__istartswith,
+        tag=tag,
+        crawl_id=crawl_id,
+        limit=limit,
+    ))
+
+
+@main.command('update')
+@click.option('--status', '-s', help='Set status')
+@click.option('--tag', '-t', help='Add tag')
+def update_cmd(status: Optional[str], tag: Optional[str]):
+    """Update Snapshots from stdin JSONL."""
+    sys.exit(update_snapshots(status=status, tag=tag))
+
+
+@main.command('delete')
+@click.option('--yes', '-y', is_flag=True, help='Confirm deletion')
+@click.option('--dry-run', is_flag=True, help='Show what would be deleted')
+def delete_cmd(yes: bool, dry_run: bool):
+    """Delete Snapshots from stdin JSONL."""
+    sys.exit(delete_snapshots(yes=yes, dry_run=dry_run))
+
+
+if __name__ == '__main__':
+    main()