release: archivebox 0.9.33rc41

release: archivebox 0.9.33rc40
release: archivebox 0.9.33rc39
2026-05-29 04:17:08 -07:00 · 2026-05-29 04:06:13 -07:00 · 2026-05-29 03:53:41 -07:00 · 2026-05-29 03:40:59 -07:00 · 2026-05-29 02:50:04 -07:00 · 2026-05-28 20:35:24 -07:00
903 changed files with 162402 additions and 32541 deletions
@@ -0,0 +1,5 @@
+{
+  "enabledPlugins": {
+    "pyright-lsp@claude-plugins-official": true
+  }
+}
@@ -0,0 +1,51 @@
+{
+  "permissions": {
+    "allow": [
+      "Read(**)",
+      "Glob(**)",
+      "Grep(**)",
+      "Bash(python -m archivebox:*)",
+      "Bash(ls:*)",
+      "Bash(xargs:*)",
+      "Bash(python -c:*)",
+      "Bash(printf:*)",
+      "Bash(pkill:*)",
+      "Bash(python3:*)",
+      "Bash(sqlite3:*)",
+      "WebFetch(domain:github.com)",
+      "Bash(uv add:*)",
+      "Bash(mkdir:*)",
+      "Bash(chmod:*)",
+      "Bash(python -m forum_dl:*)",
+      "Bash(archivebox manage migrate:*)",
+      "Bash(cat:*)",
+      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
+      "Bash(forum-dl:*)",
+      "Bash(pip uninstall:*)",
+      "Bash(python:*)",
+      "Bash(source .venv/bin/activate)",
+      "Bash(mv:*)",
+      "Bash(echo:*)",
+      "Bash(grep:*)",
+      "WebFetch(domain:python-statemachine.readthedocs.io)",
+      "Bash(./bin/run_plugin_tests.sh:*)",
+      "Bash(done)",
+      "Bash(coverage erase:*)",
+      "Bash(gh api:*)"
+    ]
+  },
+  "hooks": {
+    "PreToolUse": [
+      {
+        "matcher": "Bash",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
+            "statusMessage": "Checking working directory..."
+          }
+        ]
+      }
+    ]
+  }
+}
@@ -1,5 +1,8 @@
 .DS_Store
 ._*
+/.heartbeat.json
+/package.json
+/package-lock.json
 *.pyc
 __pycache__/
 .mypy_cache/
@@ -16,10 +19,27 @@ __pycache__/
 venv/
 .venv/
 .venv-old/
+.docker_venv/
 .docker-venv/
 node_modules/
+abx-dl/
+abxpkg/
+abx-plugins/
+abxbus/
 chrome/
 chromeprofile/
+chrome_profile/
+lib/
+out/
+users/
+archive/
+crawls/
+snapshots/
+logs/
+archivebox-docker-smoke*/
+archivebox-compose-smoke*/
+core
+*.core

 pdm.dev.lock
 pdm.lock
@@ -35,8 +55,21 @@ docker/
 website/
 typings/

+tmp/
+.tmp/
 data/
 data*/
+-
+personas/
+sources/
 output/
 index.sqlite3
 index.sqlite3-wal
+queue.sqlite3
+*.sqlite*
+data.*
+.archivebox_id
+ArchiveBox.conf
+*.stdout
+*.stderr
+*.log
@@ -0,0 +1,26 @@
+# Read the Docs config for https://docs.archivebox.io
+# https://docs.readthedocs.io/en/stable/config-file/v2.html
+
+version: 2
+
+submodules:
+  include: all
+  recursive: true
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    #nodejs: "20"     # not needed unless we need the full archivebox to run while building docs for some reason
+
+sphinx:
+  configuration: docs/conf.py
+
+formats:
+  - pdf
+  - epub
+
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt
@@ -1,2 +1,2 @@
 github: ["ArchiveBox", "pirate"]
-custom: ["https://donate.archivebox.io", "https://paypal.me/NicholasSweeting"]
+custom: ["https://donate.archivebox.io", "https://swag.archivebox.io"]
@@ -0,0 +1,198 @@
+name: 🐞 Bug report
+description: Report a bug or error you encountered in ArchiveBox
+title: "Bug: ..."
+assignees:
+  - pirate
+type: 'Bug'
+body:
+  - type: markdown
+    attributes:
+      value: |
+        *Please note:* it is normal to see errors occasionally for some extractors on some URLs (not every extractor will work on every type of page).  
+        Please report archiving errors if you are seeing them *consistently across many URLs* or if they are *preventing you from using ArchiveBox*.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Provide a screenshot and describe the bug
+      description: |
+        Attach a screenshot and describe what the issue is, what you expected to happen, and if relevant, the *URLs you were trying to archive*.  
+      placeholder: |
+        Got a bunch of 'singlefile was unable to archive this page' errors when trying to archive URLs from this site: https://example.com/xyz ...
+        I also tried to archive the same URLs using `singlefile` directly and some of them worked but not all of them. etc. ...
+    validations:
+      required: true
+
+  - type: textarea
+    id: steps_to_reproduce
+    attributes:
+      label: Steps to reproduce
+      description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
+      render: markdown
+      placeholder: |
+        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
+        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
+        3. Typed 'https://example.com/xyz' into the 'Add URL' input field
+        4. Clicked the 'Add+' button
+        5. Got a 500 error and saw the errors below in terminal
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs or errors
+      description: "Paste any terminal output, logs, or errors (check `data/logs/errors.log` as well)."
+      placeholder: |
+        ╭─────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+        │ [2024-11-02 19:54:28] ArchiveBox v0.8.6rc0: archivebox add https://example.com#1234567                  │
+        ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+        
+        [+] [2024-11-02 19:54:29] Adding 1 links to index (crawl depth=0)...
+            > Saved verbatim input to sources/1730577269-import.txt
+            > Parsed 1 URLs from input (Generic TXT)
+            ...
+      render: shell
+    validations:
+      required: false
+
+  - type: textarea
+    id: version
+    attributes:
+      label: ArchiveBox Version
+      description: |
+        **REQUIRED:** Run the `archivebox version` command inside your collection dir and paste the *full output* here (*not just the version number*).  
+        For Docker Compose run: `docker compose run archivebox version`  
+        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
+      render: shell
+      placeholder: |
+        0.8.6
+        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
+        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
+        
+        Binary Dependencies:
+        √  python     3.14.0     venv_pip   ~/.venv/bin/python
+        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
+        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
+        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
+        ...
+    validations:
+      required: true
+
+  - type: dropdown
+    id: install_method
+    validations:
+      required: true
+    attributes:
+      label: How did you install the version of ArchiveBox you are using?
+      multiple: false
+      options:
+        - pip
+        - apt
+        - brew
+        - nix
+        - Docker (or Podman/LXC/K8s/TrueNAS/Proxmox/etc)
+        - Other
+
+  - type: dropdown
+    id: operating_system
+    validations:
+      required: true
+    attributes:
+      label: What operating system are you running on?
+      description: |
+        Please note we are *unable to provide support for Windows users* unless you are using [Docker on Windows](https://github.com/ArchiveBox/archivebox#:~:text=windows%20without%20docker).
+      multiple: false
+      options:
+        - Linux (Ubuntu/Debian/Arch/Alpine/etc.)
+        - macOS (including Docker on macOS)
+        - BSD (FreeBSD/OpenBSD/NetBSD/etc.)
+        - Windows (including WSL, WSL2, Docker Desktop on Windows)
+        - Other
+
+  - type: checkboxes
+    id: filesystem
+    attributes:
+      label: What type of drive are you using to store your ArchiveBox data?
+      description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` OR `data/archive`?
+      options:
+        - label: "some of `data/` is on a local SSD or NVMe drive"
+          required: false
+        - label: "some of `data/` is on a spinning hard drive or external USB drive"
+          required: false
+        - label: "some of `data/` is on a network mount (e.g. NFS/SMB/Ceph/GlusterFS/etc.)"
+          required: false
+        - label: "some of `data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/Google Drive/Dropbox/etc.)"
+          required: false
+  
+
+  - type: textarea
+    id: docker_compose_yml
+    attributes:
+      label: Docker Compose Configuration
+      description: "If using Docker Compose, please share your full `docker-compose.yml` file. If using plain Docker, paste the `docker run ...` command you use."
+      placeholder: |
+        services:
+            archivebox:
+                image: archivebox/archivebox:latest
+                ports:
+                    - 8000:8000
+                volumes:
+                    - ./data:/data
+                environment:
+                    - ADMIN_USERNAME=admin
+                    - ADMIN_PASSWORD=****<redact any passwords>****
+                    - ALLOWED_HOSTS=*
+                    - CSRF_TRUSTED_ORIGINS=https://archivebox.example.com
+                    - PUBLIC_INDEX=True
+                    - PUBLIC_SNAPSHOTS=True
+                    - PUBLIC_ADD_VIEW=False
+                    ...
+
+            archivebox_scheduler:
+                image: archivebox/archivebox:latest
+                command: schedule --foreground --update --every=day
+                environment:
+                    ...
+      
+          ...
+      render: shell
+    validations:
+      required: false
+
+  - type: textarea
+    id: configuration
+    attributes:
+      label: ArchiveBox Configuration
+      description: "Please share your full `data/ArchiveBox.conf` file here."
+      render: shell
+      placeholder: |
+        [SERVER_CONFIG]
+        SECRET_KEY = "*********<redact any secrets/passwords>************"
+
+        WGET_RESTRICT_FILE_NAMES=windows
+        USE_SYSTEM_WGET=true
+        CHECK_SSL_VALIDITY=false
+        ...
+    validations:
+      required: false
+
+
+  - type: markdown
+    attributes:
+      value: |
+        ---
+          
+        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
+        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying anything suggested here, **we are not responsible for data loss**.  
+          
+        In the meantime please consider:  
+          
+          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
+          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
+          - 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
+          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
+        
@@ -0,0 +1,128 @@
+name: 💡 Feature or enhancement request
+description: Suggest an idea or improvement for this project
+title: "Feature Request: ..."
+assignees:
+  - pirate
+type: 'Enhancement'
+labels: 'status: idea phase'
+body:
+  - type: dropdown
+    id: suggestion_type
+    validations:
+      required: true
+    attributes:
+      label: "What type of suggestion are you making?"
+      multiple: false
+      options:
+        - New extractor / type of content to save
+        - Proposing a new feature
+        - Modification of existing behavior
+        - Web UI or UX design improvement
+
+  - type: textarea
+    id: current_problem
+    attributes:
+      label: "What is the problem that your feature request solves?"
+      description: |
+        Describe the problem or need that your feature request solves, feel free to include any screenshots or examples.
+      placeholder: |
+        e.g. I need to be able to archive spanish and french subtitle files from a particular movie site https://example.com/somevideos that's going down soon.
+    validations:
+      required: true
+
+  - type: textarea
+    id: proposed_solution
+    attributes:
+      label: "What is your proposed solution?"
+      description: |
+        Describe the ideal specific solution you'd want, *and whether it fits into any broader scope of changes*.
+      placeholder: |
+        e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
+        The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
+    validations:
+      required: true
+
+  - type: textarea
+    id: workarounds_tried
+    attributes:
+      label: "What hacks or alternative solutions have you tried to solve the problem?"
+      description: |
+        A description of any alternative approaches, workarounds, or other solutions you've considered to fix the problem.
+      placeholder: |
+        e.g. I wait for archivebox to finish archiving the page, then I manually run `yt-dlp --subs <URL>` inside 
+        the `data/archive/<timestamp>/` directory to download the subtitle files and add them to the snapshot folder.
+    validations:
+      required: true
+
+  - type: textarea
+    id: version
+    attributes:
+      label: Share the entire output of the `archivebox version` command for the current version you are using.
+      description: |
+        DO NOT JUST ENTER "the latest version" OR YOUR ISSUE WILL BE CLOSED.  
+        We need to know what version of ArchiveBox and what feature flags you're currently running with in order to contextualize your feature request.  
+        Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in your specific environment.
+          
+        Run the `archivebox version` command inside your current collection dir and paste the *full output* here (*not just the version number*).  
+        For Docker Compose run: `docker compose run archivebox version`  
+        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
+      render: shell
+      placeholder: |
+        0.8.6
+        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
+        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
+        
+        Binary Dependencies:
+        √  python     3.14.0     venv_pip   ~/.venv/bin/python
+        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
+        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
+        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
+        ...
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: priority
+    attributes:
+      label: "How badly do you want this new feature?"
+      options:
+        - label: "It's an urgent deal-breaker, I can't live without it"
+          required: false
+        - label: "It's important to add it in the near-mid term future"
+          required: false
+        - label: "It would be nice to have eventually"
+          required: false
+        - label: "I'm willing to [start a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
+          required: false
+        - label: "I have [donated money](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations) to go towards fixing this issue"
+          required: false
+  
+  - type: checkboxes
+    id: satisfaction_survey
+    attributes:
+      label: Mini Survey
+      description: How do you like ArchiveBox so far?
+      options:
+        - label: "I like ArchiveBox so far / would recommend it to a friend"
+          required: false
+        - label: "I've had a lot of difficulty getting ArchiveBox set up"
+          required: false
+        - label: "I would pay $10/mo for a hosted version of ArchiveBox if it had this feature"
+          required: false
+
+  - type: markdown
+    attributes:
+      value: |
+        ---
+          
+        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
+        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying any workarounds or BETAs suggested here, **we are not responsible for data loss**.  
+          
+        In the meantime please consider:  
+          
+          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 📊 [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
+          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
+          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
@@ -0,0 +1,52 @@
+name: 📑 Documentation improvement
+description: Submit an idea or correction for the Wiki documentation
+title: "Documentation: ..."
+labels: 'touches: docs'
+type: 'Enhancement'
+assignees:
+  - pirate
+body:
+  - type: markdown
+    attributes:
+      value: |
+        If you prefer, you can submit a [Pull Request](https://github.com/ArchiveBox/docs) on https://github.com/ArchiveBox/docs to edit the docs directly instead.
+
+  - type: input
+    id: page_url
+    validations:
+      required: true
+    attributes:
+      label: "What is the URL of the page you'd like to see improved?"
+      placeholder: e.g. https://github.com/ArchiveBox/docs/wiki/Install
+
+  - type: input
+    id: section_title
+    validations:
+      required: true
+    attributes:
+      label: "What is the title of the relevant section?"
+      placeholder: e.g. Option B. Automatic Setup Script
+
+  - type: textarea
+    id: suggested_edit
+    attributes:
+      label: "What is the suggested edit?"
+      placeholder: |
+        e.g. Please document how to run the automatic setup script for ArchiveBox on TempleOS.  
+        Attach images, screenshots, code snippets, etc. anything you think would help.
+    validations:
+      required: true
+
+  - type: markdown
+    attributes:
+      value: |
+        ---
+          
+        We strive to address issues as quickly as possible, it usually takes us *about a ~week* to respond.  
+          
+        In the meantime please consider:  
+          
+          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
+          - 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
+          - ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)
@@ -1,46 +0,0 @@
---
-name: 🐞 Bug report
-about: Create a report to help us improve
-title: 'Bug: ...'
-labels: 'bug'
-assignees: ''
-
---
-
-<!--
-Please fill out the following information, 
-feel free to delete sections if they're not applicable 
-or if long issue templates annoy you.
-(the only required section is the version information)
-->
-
-#### Describe the bug
-<!--
-A description of what the bug is, 
-what you expected to happen, 
-and any relevant context about issue.
-->
-
-#### Steps to reproduce
-<!--
-For example:
-1. Ran ArchiveBox with the following config '...'
-2. Saw this output during archiving '....'
-3. UI didn't show the thing I was expecting '....'
-->
-
-#### Screenshots or log output
-
-<!--
-If applicable, post any relevant screenshots or copy/pasted terminal output from ArchiveBox.
-If you're reporting a parsing / importing error, **you must paste a copy of your redacted import file here**.
-->
-
-#### ArchiveBox version
-
-<!-- Run the `archivebox version` command locally then copy paste the result here: -->
-```logs
-replace this line with the *full*, unshortened output of running `archivebox version`
-```
-<!-- Tickets without full version info will closed until it is provided,
-we need the full output here to help you solve your issue -->
@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: ❓ Ask a question or start a discussion
+    url: https://github.com/ArchiveBox/ArchiveBox/discussions
+    about: "Ask a question, get support, or start a design discussion (to report a problem please use '🐞 Bug report' instead)"
+  - name: 💬 Chat with the dev team & community on Zulip
+    url: https://zulip.archivebox.io
+    about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
+  - name: 💁‍♂️ Hire us for professional support with fast response times
+    url: https://docs.monadical.com/s/archivebox-consulting-services
+    about: "We provide hosting, development, and support, including on-prem/cloud w/ SSO & storage, CAPTCHA-solving, proxies, etc."
@@ -1,17 +0,0 @@
---
-name: 📑 Documentation change
-about: Submit a suggestion for the Wiki documentation
-title: 'Documentation: Improvement request ...'
-labels: ''
-assignees: ''
-
---
-<!-- If you perfer, you can make a PR to https://github.com/ArchiveBox/docs instead of opening an issue -->
-
-## Wiki Page URL
-<!-- e.g. https://github.com/pirate/ArchiveBox/wiki/Configuration#use_color -->
-
-
-## Suggested Edit
-<!-- e.g. Please add more example usages, or please fix `xyz` typo to be `abc`. -->
-
@@ -1,50 +0,0 @@
---
-name: 💡 Feature request
-about: Suggest an idea for this project
-title: 'Feature Request: ...'
-labels: 'changes: behavior,status: idea phase'
-assignees: ''
-
---
-
-<!--
-Please fill out the following information, 
-feel free to delete sections if they're not applicable 
-or if long issue templates annoy you :)
-->
-
-## Type 
-
- - [ ] General question or discussion
- - [ ] Propose a brand new feature
- - [ ] Request modification of existing behavior or design
-
-## What is the problem that your feature request solves
-<!--
-e.g. I need to be able to archive spanish and french subtitle files  
-from a particular <example.com> movie site that's going down soon.
-->
-
-## Describe the ideal specific solution you'd want, and whether it fits into any broader scope of changes
-<!--
-e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
-The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
-->
-
-## What hacks or alternative solutions have you tried to solve the problem?
-<!--
-A clear and concise description of any alternative solutions, 
-workarounds, or other software you've considered using to fix the problem.
-->
-
-## How badly do you want this new feature?
-
- - [ ] It's an urgent deal-breaker, I can't live without it
- - [ ] It's important to add it in the near-mid term future
- - [ ] It would be nice to have eventually
-
---
-
- - [ ] I'm willing to contribute [dev time](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) / [money](https://github.com/sponsors/pirate) to fix this issue
- - [ ] I like ArchiveBox so far / would recommend it to a friend
- - [ ] I've had a lot of difficulty getting ArchiveBox set up
@@ -1,9 +0,0 @@
---
-name: 💬 Question, discussion, or support request
-about: Start a discussion or ask a question about ArchiveBox
-title: 'Question: ...'
-labels: ''
-assignees: ''
-
---
-
@@ -0,0 +1,49 @@
+name: Claude Code
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  issues:
+    types: [opened, assigned]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  claude:
+    if: |
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
+      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write
+      id-token: write
+      actions: read # Required for Claude to read CI results on PRs
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@v1
+        with:
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+
+          # This is an optional setting that allows Claude to read CI results on PRs
+          additional_permissions: |
+            actions: read
+
+          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
+          # prompt: 'Update the pull request description to include a summary of changes.'
+
+          # Optional: Add claude_args to customize behavior and configuration
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
+          claude_args: '--allowed-tools Bash(gh pr:*)'
@@ -2,63 +2,209 @@ name: Build Debian package

 on:
  workflow_dispatch:
+  workflow_call:
  push:
+    branches: ['**']
+    paths:
+      - 'pkg/debian/**'
+      - 'bin/build_deb.sh'
+      - 'bin/release_deb.sh'
+      - '.github/workflows/debian.yml'
+      - 'pyproject.toml'
+  # release trigger is handled by release.yml to avoid double-runs

-env:
-  DEB_BUILD_OPTIONS: nocheck
+permissions:
+  contents: write

 jobs:
  build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        arch: [amd64, arm64]

    steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
        with:
-          submodules: true
          fetch-depth: 1

-      - name: Install packaging dependencies
+      - name: Get version
+        id: version
+        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
+
+      - name: Install nfpm
+        env:
+          NFPM_VERSION: "2.45.1"
        run: |
-          sudo apt-get update -qq
-          sudo apt-get install -y \
-            python3 python3-dev python3-pip python3-venv python3-all \
-            dh-python debhelper devscripts dput software-properties-common \
-            python3-distutils python3-setuptools python3-wheel python3-stdeb
+          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
+            | sudo tar -xz -C /usr/local/bin nfpm
+          nfpm --version

-      # - name: Build Debian/Apt sdist_dsc
-      #   run: |
-      #     ./bin/build_pip.sh
+      - name: Build .deb package
+        run: |
+          export VERSION="${{ steps.version.outputs.version }}"
+          export ARCH="${{ matrix.arch }}"
+          ./bin/build_deb.sh

-      # - name: Check ArchiveBox version
-      #   run: |
-      #     # must create dir needed for snaps to run as non-root on github actions
-      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
-      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
-      #     archivebox --version
-      #     archivebox init --setup
+      - name: Verify .deb package contents
+        run: |
+          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
+          echo "=== Package info ==="
+          dpkg-deb --info "$DEB_FILE"
+          echo ""
+          echo "=== Package contents ==="
+          dpkg-deb --contents "$DEB_FILE"
+          echo ""
+          echo "=== Control fields ==="
+          dpkg-deb --field "$DEB_FILE"

-      # - name: Add some links to test
-      #   run: |
-      #     cd "${{ github.workspace }}/data"
-      #     archivebox add 'https://example.com'
-      #     archivebox status
+      - name: Upload .deb artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
+          path: dist/*.deb

-      # - name: Commit built package
-      #   run: |
-      #     cd deb_dist/
-      #     git config --local user.email "action@github.com"
-      #     git config --local user.name "GitHub Action"
-      #     git commit -m "Debian package autobuild" -a
-      
-      # - name: Push build to Github
-      #   uses: ad-m/github-push-action@master
-      #   with:
-      #     github_token: ${{ secrets.GITHUB_TOKEN }}
-      #     repository: ArchiveBox/debian-archivebox
-      #     branch: ${{ github.ref }}
-      #     directory: deb_dist
+  test:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-24.04
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}

-      # - name: Push build to Launchpad PPA
-      #   run: |
-      #     debsign -k "$PGP_KEY_ID" "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
-      #     dput archivebox "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: false
+
+      - name: Install build dependencies
+        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
+        with:
+          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
+          version: 1.0
+
+      - name: Build local wheel
+        run: |
+          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+          uv build --wheel --out-dir /tmp/wheels/
+
+      - name: Download .deb artifact
+        uses: actions/download-artifact@v4
+        with:
+          pattern: archivebox-*-${{ matrix.arch }}.deb
+          merge-multiple: true
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
+
+      - name: Pre-seed virtualenv with local wheel before dpkg install
+        run: |
+          # CI-only: pre-seed the venv with a local wheel so we test the
+          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
+          # here (matching the system dep installed above) to ensure the venv is
+          # created with the correct Python version. On real installs, install.sh
+          # handles this by preferring python3.13 and failing if < 3.13.
+          # When postinstall.sh runs during dpkg -i, it finds the venv already
+          # populated and just upgrades deps.
+          sudo mkdir -p /opt/archivebox
+          sudo python3.13 -m venv /opt/archivebox/venv
+          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
+          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
+          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
+
+      - name: Install .deb package
+        run: |
+          # dpkg install will run postinstall which creates the user,
+          # sets up systemd, and tries pip install (which finds it already installed)
+          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
+
+      - name: Verify archivebox is installed
+        run: |
+          which archivebox
+          archivebox version
+
+      - name: Verify wrapper script works
+        run: |
+          /usr/bin/archivebox version
+          /usr/bin/archivebox --help | head -5
+
+      - name: Test archivebox init as archivebox user
+        run: |
+          # The postinstall should have created the user
+          id archivebox
+          sudo mkdir -p /tmp/archivebox-test
+          sudo chown archivebox:archivebox /tmp/archivebox-test
+          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
+
+      - name: Test archivebox status
+        run: |
+          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
+
+      - name: Test archivebox add
+        run: |
+          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
+
+      - name: Verify systemd service file exists
+        run: |
+          test -f /usr/lib/systemd/system/archivebox.service
+          cat /usr/lib/systemd/system/archivebox.service
+
+  # Upload .deb to GitHub Release only after tests pass
+  release:
+    if: github.event_name == 'release' || github.event_name == 'workflow_call'
+    needs: [build, test]
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Determine release tag
+        id: tag
+        run: |
+          TAG="${{ github.event.release.tag_name }}"
+          if [ -z "$TAG" ]; then
+            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
+            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Download all .deb artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: archivebox-*.deb
+          merge-multiple: true
+
+      - name: Upload .deb to GitHub Release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          TAG="${{ steps.tag.outputs.tag }}"
+          # Verify the release exists before uploading
+          if ! gh release view "$TAG" >/dev/null 2>&1; then
+            echo "[!] No GitHub Release found for tag $TAG."
+            if [ -n "${{ github.event.release.tag_name }}" ]; then
+              echo "[X] This was triggered by a release event — the release should exist. Failing."
+              exit 1
+            fi
+            echo "[i] Skipping upload (workflow_dispatch without a release)."
+            echo "    Create a release first or trigger via the release event."
+            exit 0
+          fi
+          gh release upload "$TAG" *.deb --clobber
@@ -0,0 +1,43 @@
+name: Deploy Publicsite to GitHub Pages
+
+on:
+  push:
+    branches:
+      - dev
+    paths:
+      - publicsite/**
+      - .github/workflows/deploy-publicsite.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: github-pages-publicsite
+  cancel-in-progress: true
+
+jobs:
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./publicsite
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
@@ -2,28 +2,48 @@ name: Build Docker image

 on:
  workflow_dispatch:
+  workflow_call:
  push:
    branches:
-      - '**'
+      - dev
    tags:
      - 'v*'
  # pull_request:

 env:
-  DOCKER_IMAGE: archivebox-ci
+  DOCKERHUB_IMAGE: archivebox/archivebox
+  GHCR_IMAGE: ghcr.io/archivebox/archivebox
+
+permissions:
+  contents: read
+  packages: write
+
+concurrency:
+  group: docker-${{ github.ref }}
+  cancel-in-progress: true

 jobs:
-  buildx:
-    runs-on: ubuntu-latest
+  build:
+    name: build ${{ matrix.platform }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-24.04
+            cache_scope: docker-amd64
+            artifact_name: digest-linux-amd64
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
+            cache_scope: docker-arm64
+            artifact_name: digest-linux-arm64
    steps:
      - name: Checkout
        uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        # with:
+        #   submodules: true
+        #   fetch-depth: 1

      - name: Set up Docker Buildx
        id: buildx
@@ -31,7 +51,6 @@ jobs:
        with:
          version: latest
          install: true
-          platforms: linux/amd64,linux/arm64

      - name: Builder instance name
        run: echo ${{ steps.buildx.outputs.name }}
@@ -39,69 +58,204 @@ jobs:
      - name: Available platforms
        run: echo ${{ steps.buildx.outputs.platforms }}

-      - name: Cache Docker layers
-        uses: actions/cache@v3
-        with:
-          path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-
-
-      - name: Docker Login
+      - name: Login to Docker Hub
        uses: docker/login-action@v3
        if: github.event_name != 'pull_request'
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

-      - name: Collect Docker tags
-        # https://github.com/docker/metadata-action
-        id: docker_meta
-        uses: docker/metadata-action@v5
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
        with:
-          images: archivebox/archivebox,nikisweeting/archivebox
-          tags: |
-              # :stable
-              type=ref,event=branch
-              # :0.7.3
-              type=semver,pattern={{version}}
-              # :0.7
-              type=semver,pattern={{major}}.{{minor}}
-              # :sha-463ea54
-              type=sha
-              # :latest
-              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}

-      - name: Build and push
+      - name: Collect Docker labels
+        id: docker_meta
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(python3 - <<'PY'
+          from pathlib import Path
+          import re
+
+          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
+          if not match:
+              raise SystemExit("Failed to read version from pyproject.toml")
+          print(match.group(1))
+          PY
+          )"
+
+          {
+            echo 'labels<<EOF'
+            echo "org.opencontainers.image.version=${VERSION}"
+            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
+            echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
+            echo 'EOF'
+            echo "version=${VERSION}"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "[+] Building ${{ matrix.platform }} for ${VERSION}"
+
+      - name: Build and push digest
        id: docker_build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
        with:
          context: ./
          file: ./Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.docker_meta.outputs.tags }}
+          tags: |
+            ${{ env.DOCKERHUB_IMAGE }}
+            ${{ env.GHCR_IMAGE }}
          labels: ${{ steps.docker_meta.outputs.labels }}
-          cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache-new
-          platforms: linux/amd64,linux/arm64
+          cache-from: type=gha,scope=${{ matrix.cache_scope }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
+          platforms: ${{ matrix.platform }}
+          outputs: type=image,push-by-digest=true,name-canonical=true,push=true

      - name: Image digest
        run: echo ${{ steps.docker_build.outputs.digest }}

+      - name: Export digest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mkdir -p /tmp/digests
+          digest="${{ steps.docker_build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.artifact_name }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  publish:
+    name: publish multiarch tags
+    runs-on: ubuntu-24.04
+    needs:
+      - build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: latest
+          install: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digest-*
+          merge-multiple: true
+
+      - name: Collect Docker tags
+        id: docker_meta
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(python3 - <<'PY'
+          from pathlib import Path
+          import re
+
+          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
+          if not match:
+              raise SystemExit("Failed to read version from pyproject.toml")
+          print(match.group(1))
+          PY
+          )"
+          SHORT_SHA="${GITHUB_SHA::8}"
+
+          {
+            echo 'dockerhub_tags<<EOF'
+            echo "${DOCKERHUB_IMAGE}:dev"
+            echo "${DOCKERHUB_IMAGE}:${VERSION}"
+            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+            echo 'EOF'
+            echo 'ghcr_tags<<EOF'
+            echo "${GHCR_IMAGE}:dev"
+            echo "${GHCR_IMAGE}:${VERSION}"
+            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+            echo 'EOF'
+            echo "version=${VERSION}"
+            echo "short_sha=${SHORT_SHA}"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "[+] Publishing Docker Hub tags:"
+          printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+          echo "[+] Publishing GHCR tags:"
+          printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+
+      - name: Create Docker Hub manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
+          [[ "${#DIGESTS[@]}" -gt 0 ]]
+
+          TAG_ARGS=()
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do
+            REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}")
+          done
+
+          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Create GHCR manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
+          [[ "${#DIGESTS[@]}" -gt 0 ]]
+
+          TAG_ARGS=()
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
+
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do
+            REFS+=("${GHCR_IMAGE}@sha256:${digest}")
+          done
+
+          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Inspect published images
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+          docker buildx imagetools inspect "${GHCR_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+
      - name: Update README
        uses: peter-evans/dockerhub-description@v4
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
          repository: archivebox/archivebox
-
-      # This ugly bit is necessary if you don't want your cache to grow forever
-      # until it hits GitHub's limit of 5GB.
-      # Temp fix
-      # https://github.com/docker/build-push-action/issues/252
-      # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
@@ -0,0 +1,59 @@
+name: Duplicate Issue Detection
+
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  check-duplicates:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Install opencode
+        run: curl -fsSL https://opencode.ai/install | bash
+
+      - name: Check for duplicate issues
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENCODE_PERMISSION: |
+            {
+              "bash": {
+                "gh issue*": "allow",
+                "gh pr*": "allow",
+                "*": "deny"
+              },
+              "webfetch": "allow"
+            }
+        run: |
+          opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created:
+
+          Issue number:
+          ${{ github.event.issue.number }}
+
+          Lookup this issue and search through existing issues and PRs (excluding #${{ github.event.issue.number }}) in this repository to find any potential duplicates of this new issue.
+          Consider:
+          1. Similar titles or descriptions
+          2. Same error messages or symptoms
+          3. Related functionality or components
+          4. Similar feature requests
+
+          If you find any potential duplicates, please comment on the new issue with:
+          - A brief explanation of why it might be a duplicate
+          - Links to the potentially duplicate issues or PRs
+          - A suggestion to check those issues first
+
+          Use this format for the comment:
+          'This issue might be a duplicate of existing issues. Please check:
+          - #[issue_number]: [brief description of similarity]
+
+          Feel free to ignore if none of these address your specific case.'
+
+          If no clear duplicates are found, do not comment."
@@ -1,43 +0,0 @@
-# Simple workflow for deploying static content to GitHub Pages
-name: Deploy static content to Pages
-
-on:
-  # Runs on pushes targeting the default branch
-  push:
-    branches: ["dev"]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
-# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
-concurrency:
-  group: "pages"
-  cancel-in-progress: false
-
-jobs:
-  # Single deploy job since we're just deploying
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Setup Pages
-        uses: actions/configure-pages@v5
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
-        with:
-          # Upload entire repository
-          path: './website'
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4
@@ -1,51 +1,229 @@
-name: Build Homebrew package
+name: Build Homebrew formula

 on:
  workflow_dispatch:
+  workflow_call:
  push:
+    branches: ['**']
+    paths:
+      - 'brew_dist/**'
+      - 'bin/build_brew.sh'
+      - 'bin/release_brew.sh'
+      - '.github/workflows/homebrew.yml'
+      - 'pyproject.toml'
+  # release trigger is handled by release.yml to avoid double-runs

+permissions:
+  contents: read

 jobs:
-  build:
+  build-and-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-24.04]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Get version
+        id: version
+        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
+
+      - name: Validate formula template syntax
+        run: ruby -c brew_dist/archivebox.rb
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: false
+
+      - name: Install build dependencies (Linux)
+        if: runner.os == 'Linux'
+        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
+        with:
+          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
+          version: 1.0
+
+      - name: Build local sdist
+        run: |
+          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+          uv build --sdist --out-dir /tmp/sdist/
+
+      - name: Generate formula from local sdist
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
+          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
+
+          # Install archivebox + poet into a temp venv to generate resource stanzas
+          python3 -m venv /tmp/poet-venv
+          source /tmp/poet-venv/bin/activate
+          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
+          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
+          RESOURCES="$(poet archivebox)"
+          deactivate
+
+          # For CI: use file:// URL pointing to local sdist
+          # For release: this gets overridden with the PyPI URL
+          SDIST_URL="file://${SDIST_PATH}"
+
+          cat > /tmp/archivebox.rb << RUBY
+class Archivebox < Formula
+  include Language::Python::Virtualenv
+
+  desc "Self-hosted internet archiving solution"
+  homepage "https://github.com/ArchiveBox/ArchiveBox"
+  url "${SDIST_URL}"
+  sha256 "${SDIST_SHA256}"
+  license "MIT"
+
+  depends_on "python@3.13"
+
+${RESOURCES}
+
+  def install
+    virtualenv_install_with_resources
+  end
+
+  def post_install
+    data_dir = var/"archivebox"
+    data_dir.mkpath
+    ENV["DATA_DIR"] = data_dir.to_s
+    system bin/"archivebox", "init"
+  end
+
+  def caveats
+    <<~EOS
+      ArchiveBox data is stored in:
+        #{var}/archivebox
+
+      To start archiving, run:
+        cd #{var}/archivebox && archivebox add 'https://example.com'
+
+      To start the web UI:
+        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
+    EOS
+  end
+
+  test do
+    assert_match version.to_s, shell_output("#{bin}/archivebox version")
+  end
+end
+RUBY
+
+          echo "[√] Generated formula:"
+          ruby -c /tmp/archivebox.rb
+          cat /tmp/archivebox.rb
+
+      - name: Install Homebrew (Linux only)
+        if: runner.os == 'Linux'
+        run: |
+          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
+          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
+          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
+          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
+          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
+          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
+
+      - name: Install brew dependencies
+        run: brew install python@3.13
+
+      - name: Install archivebox via brew from local formula
+        run: |
+          brew install --build-from-source --verbose /tmp/archivebox.rb
+
+      - name: Verify archivebox version
+        run: archivebox version
+
+      - name: Test archivebox init
+        run: |
+          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
+          archivebox init
+
+      - name: Test archivebox status
+        run: |
+          cd /tmp/archivebox-test
+          archivebox status
+
+  # On release only: generate the real formula with PyPI URL and push to tap
+  release:
+    if: github.event_name == 'release' || github.event_name == 'workflow_call'
+    needs: build-and-test
    runs-on: macos-latest

    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
        with:
-          submodules: true
          fetch-depth: 1

-      # TODO: modify archivebox.rb to update src url, hashes, and dependencies
+      - name: Get version
+        id: version
+        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"

-      - name: Build Homebrew Bottle
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Wait for PyPI package availability
        run: |
-          pip3 install --upgrade pip setuptools wheel
-          cd brew_dist/
-          brew install --build-bottle ./archivebox.rb
-          # brew bottle archivebox
+          VERSION="${{ steps.version.outputs.version }}"
+          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
+          for i in $(seq 1 30); do
+            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
+              echo "[√] archivebox==${VERSION} is available on PyPI"
+              break
+            fi
+            if [ "$i" -eq 30 ]; then
+              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
+              break
+            fi
+            echo "    Attempt $i/30 - not yet available, waiting 30s..."
+            sleep 30
+          done
+
+      - name: Generate release formula via build_brew.sh
+        run: ./bin/build_brew.sh
+
+      - name: Test formula install
+        run: |
+          brew install --build-from-source brew_dist/archivebox.rb
          archivebox version

-      - name: Add some links to test
+      - name: Upload formula artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: archivebox.rb
+          path: brew_dist/archivebox.rb
+
+      - name: Push to homebrew-archivebox tap
+        env:
+          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
        run: |
-          mkdir data && cd data
-          archivebox init --setup
-          archivebox add 'https://example.com'
-          archivebox version
-          archivebox status
+          VERSION="${{ steps.version.outputs.version }}"
+          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap

-      # - name: Commit built package
-      #   run: |
-      #     cd brew_dist/
-      #     git config --local user.email "action@github.com"
-      #     git config --local user.name "GitHub Action"
-      #     git commit -m "Homebrew package autobuild" -a
-      
-      # - name: Push build to Github
-      #   uses: ad-m/github-push-action@master
-      #   with:
-      #     github_token: ${{ secrets.GITHUB_TOKEN }}
-      #     repository: ArchiveBox/homebrew-archivebox
-      #     branch: ${{ github.ref }}
-      #     directory: brew_dist
+          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
+            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb

-      # TODO: push bottle homebrew core PR with latest changes
+          cd /tmp/tap
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add -A
+          if git diff --cached --quiet; then
+            echo "[i] No changes to formula, skipping push."
+          else
+            git commit -m "Update archivebox to v${VERSION}"
+            git push origin HEAD
+            echo "[√] Formula pushed to homebrew-archivebox tap"
+          fi
@@ -5,31 +5,42 @@ on:
  push:

 env:
-  MAX_LINE_LENGTH: 110
+  UV_NO_SOURCES: "1"
+  PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl

 jobs:
  lint:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1

+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
      - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
        with:
-          python-version: 3.9
+          python-version: "3.13"
          architecture: x64

-      - name: Install flake8
-        run: |
-          pip install flake8
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: false

-      - name: Lint with flake8
+      - name: Install dependencies with uv
        run: |
-          cd archivebox
-          # one pass for show-stopper syntax errors or undefined names
-          flake8 . --count --show-source --statistics
-          # one pass for small stylistic things
-          flake8 . --count --max-line-length="$MAX_LINE_LENGTH" --statistics
+          uv sync --all-extras --all-groups --no-sources --no-cache
+          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+
+      - name: Run prek
+        run: uv run --no-sync --no-sources prek run --all-files
@@ -2,77 +2,68 @@ name: Build Pip package

 on:
  workflow_dispatch:
+  workflow_call:
  push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'

+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHON_VERSION: "3.13"

 jobs:
  build:
-    runs-on: ubuntu-22.04
-
    permissions:
      id-token: write

+    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
        with:
-          submodules: true
-          fetch-depth: 1
+          enable-cache: false

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
-          python-version: '3.11'
+          python-version: ${{ env.PYTHON_VERSION }}
          architecture: x64

-      - uses: awalsh128/cache-apt-pkgs-action@latest
+      - name: APT install archivebox dev + run dependencies
+        uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

-      - uses: pdm-project/setup-pdm@v3
+      - name: UV install archivebox dev + run sub-dependencies
+        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+
+      - name: UV build archivebox and archivebox/pkgs/* packages
+        run: |
+          uv build --all
+
+      - name: Publish new package wheels and sdists to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # ignore when publish to PyPI fails due to duplicate tag
+        continue-on-error: true
        with:
-          python-version: '3.11'
-          cache: true
+          password: ${{ secrets.PYPI_PAT_SECRET }}

-      - name: Install dependencies
-        run: pdm install --fail-fast --no-lock --dev --group=':all' --no-self
+      - name: UV install archivebox and archivebox/pkgs/* locally for tests
+        run: uv sync --all-extras --no-sources --no-cache

-      - name: Build package
+      - name: UV run archivebox init + archivebox version
        run: |
-          pdm build
-
-      - name: Install from build
-        run: pip install ./dist/archivebox-*.whl
-
-      - name: Add some links to test
-        run: |
-          mkdir data && cd data
-          archivebox init
-          archivebox add 'https://example.com'
-          archivebox version
-          archivebox status
-
-      #- name: Publish package distributions to PyPI
-      #  run: pdm publish --no-build
-
-      # - name: Push build to PyPI
-      #   run: |
-      #     cd pip_dist/
-      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
-      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
-
-      # - name: Commit built package
-      #   run: |
-      #     cd pip_dist/
-      #     git config --local user.email "action@github.com"
-      #     git config --local user.name "GitHub Action"
-      #     git commit -m "Pip package autobuild" -a
-      
-      # - name: Push build to Github
-      #   uses: ad-m/github-push-action@master
-      #   with:
-      #     github_token: ${{ secrets.GITHUB_TOKEN }}
-      #     repository: ArchiveBox/pip-archivebox
-      #     branch: ${{ github.ref }}
-      #     directory: pip_dist
-
+          mkdir -p data && cd data
+          uv run --no-sync --no-sources archivebox init \
+          && uv run --no-sync --no-sources archivebox version
+          # && uv run archivebox add 'https://example.com' \
+          # && uv run archivebox status \
+          # || (echo "UV Failed to run archivebox!" && exit 1)
@@ -0,0 +1,45 @@
+name: Release State
+
+on:
+  push:
+    branches:
+      - '**'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  id-token: write
+
+jobs:
+  release-state:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+          ref: ${{ github.ref_name }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: false
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Configure git identity
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Run release script
+        env:
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+          GH_TOKEN: ${{ github.token }}
+          PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
+        run: ./bin/release.sh
@@ -0,0 +1,43 @@
+name: Release
+
+# Orchestrates the full release pipeline:
+#   1. Build and publish pip package to PyPI
+#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
+#   3. Build Docker images (in parallel with deb/brew)
+#
+# Individual workflows also run on push for CI (see their own triggers).
+# Manual fallback for the hosted release pipeline. The normal prerelease loop
+# is run by bin/release.sh locally so published GitHub releases do not launch a
+# duplicate release job.
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  packages: write
+  id-token: write
+
+jobs:
+  pip:
+    name: Publish to PyPI
+    uses: ./.github/workflows/pip.yml
+    secrets: inherit
+
+  debian:
+    name: Build .deb packages
+    needs: pip
+    uses: ./.github/workflows/debian.yml
+    secrets: inherit
+
+  homebrew:
+    name: Update Homebrew formula
+    needs: pip
+    uses: ./.github/workflows/homebrew.yml
+    secrets: inherit
+
+  docker:
+    name: Build Docker images
+    needs: pip
+    uses: ./.github/workflows/docker.yml
+    secrets: inherit
@@ -0,0 +1,296 @@
+name: Parallel Tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [dev, main, master]
+  push:
+    branches: [dev]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHONIOENCODING: utf-8
+  PYTHONLEGACYWINDOWSSTDIO: utf-8
+  USE_COLOR: False
+  UV_NO_SOURCES: "1"
+
+jobs:
+  discover-tests:
+    name: Discover test files
+    runs-on: ubuntu-24.04
+    outputs:
+      test-files: ${{ steps.set-matrix.outputs.test-files }}
+      plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Discover test files
+        id: set-matrix
+        run: |
+          # Find all main test files
+          all_tests=$(find archivebox/tests -maxdepth 1 -name "test_*.py" -type f | sort)
+
+          # Create JSON array with test file info
+          json_array="["
+          first=true
+          for test_file in $all_tests; do
+            if [ "$first" = true ]; then
+              first=false
+            else
+              json_array+=","
+            fi
+
+            # Extract a display name for the test
+            name="main/$(basename $test_file .py | sed 's/^test_//')"
+
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+          done
+          json_array+="]"
+
+          echo "test-files=$json_array" >> $GITHUB_OUTPUT
+          echo "Found $(echo $all_tests | wc -w) test files"
+          echo "$json_array" | jq '.'
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Discover plugin tests
+        id: set-plugin-matrix
+        run: |
+          plugin_tests=$(find abx-plugins/abx_plugins/plugins -maxdepth 2 -type d -name tests | sed 's#abx-plugins/abx_plugins/plugins/##; s#/tests##' | sort)
+
+          json_array="["
+          first=true
+          for plugin_name in $plugin_tests; do
+            if [ "$first" = true ]; then
+              first=false
+            else
+              json_array+=","
+            fi
+
+            needs_chromium=false
+            if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
+              needs_chromium=true
+            fi
+
+            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium}"
+          done
+          json_array+="]"
+
+          echo "plugin-tests=$json_array" >> $GITHUB_OUTPUT
+          echo "Found $(echo $plugin_tests | wc -w) plugin test suites"
+          echo "$json_array" | jq '.'
+
+  run-tests:
+    name: ${{ matrix.test.name }}
+    runs-on: ubuntu-24.04
+    needs: discover-tests
+    env:
+      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
+      CHROME_HEADLESS: "true"
+      CHROME_SANDBOX: "false"
+      PERSONAS_DIR: /tmp/abx-personas
+      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
+
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
+        python: ["3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "latest"
+
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Cache uv
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python }}-uv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.1
+
+      - name: Install dependencies with uv
+        run: |
+          uv sync --dev --all-extras --no-sources --no-cache
+          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+
+      - name: Run test - ${{ matrix.test.name }}
+        run: |
+          mkdir -p tests/out
+          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
+
+  plugin-tests:
+    name: ${{ matrix.plugin.name }}
+    runs-on: ubuntu-24.04
+    needs: discover-tests
+    env:
+      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
+      CHROME_HEADLESS: "true"
+      CHROME_SANDBOX: "false"
+      PERSONAS_DIR: /tmp/abx-personas
+      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
+
+    strategy:
+      fail-fast: false
+      matrix:
+        plugin: ${{ fromJson(needs.discover-tests.outputs.plugin-tests) }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          architecture: x64
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "latest"
+
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Cache uv
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-3.13-uv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.1
+
+      - name: Install dependencies with uv
+        run: |
+          uv sync --dev --all-extras --no-sources --no-cache
+          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+
+      - name: Select Chromium for extension-capable CDP
+        if: ${{ matrix.plugin.needs_chromium }}
+        env:
+          MIN_CHROMIUM_MAJOR: "149"
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          candidate="/usr/bin/chromium"
+          version=""
+          major=0
+          if [ -x "$candidate" ]; then
+            version="$("$candidate" --version || true)"
+            major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+            case "$major" in
+              ''|*[!0-9]*) major=0 ;;
+            esac
+          fi
+
+          if [ "$major" -ge "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "Using runner Chromium: $candidate ($version)"
+            echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+            exit 0
+          fi
+
+          echo "Runner Chromium is unavailable or too old: ${version:-not found}"
+          echo "Installing Playwright Chromium into $LIB_DIR..."
+          uv run --no-sync --no-sources abxpkg install \
+            --binproviders=npm \
+            --lib="$LIB_DIR" \
+            --postinstall-scripts=false \
+            --min-release-age=0 \
+            --no-cache \
+            --overrides='{"npm":{"install_args":["playwright@next"]}}' \
+            playwright
+          uv run --no-sync --no-sources abxpkg install \
+            --binproviders=playwright \
+            --lib="$LIB_DIR" \
+            --bin-dir="$LIB_DIR/env/bin" \
+            --min-version="$MIN_CHROMIUM_MAJOR" \
+            chromium
+
+          candidate="$LIB_DIR/env/bin/chromium"
+          if [ ! -x "$candidate" ]; then
+            echo "Playwright Chromium shim was not created at $candidate" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+          case "$major" in
+            ''|*[!0-9]*) major=0 ;;
+          esac
+          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "Playwright Chromium is still too old: ${version:-unknown}" >&2
+            exit 1
+          fi
+          echo "Using Playwright Chromium: $candidate ($version)"
+          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+
+      - name: Run plugin tests - ${{ matrix.plugin.name }}
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
+        run: |
+          uv run --no-sync --no-sources bash ./bin/test_plugins.sh "${{ matrix.plugin.plugin }}" --no-coverage
@@ -6,16 +6,19 @@ env:
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False
+  UV_NO_SOURCES: "1"

 jobs:
  python_tests:
    runs-on: ${{ matrix.os }}
+    env:
+      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl

    strategy:
      matrix:
        os: [ubuntu-22.04]
        # os: [ubuntu-22.04, macos-latest, windows-latest]
-        python: [3.11]
+        python: ["3.13"]

    steps:
      - uses: actions/checkout@v4
@@ -23,6 +26,15 @@ jobs:
          submodules: true
          fetch-depth: 1

+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
      ### Setup Python & JS Languages
      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@v4
@@ -30,63 +42,34 @@ jobs:
          python-version: ${{ matrix.python }}
          architecture: x64

+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
      - name: Set up Node JS
        uses: actions/setup-node@v4
        with:
-          node-version: 20.10.0
-
-      - name: Setup PDM
-        uses: pdm-project/setup-pdm@v3
-        with:
-          python-version: '3.11'
-          cache: true
+          node-version: 22

      ### Install Python & JS Dependencies
-      - name: Get pip cache dir
-        id: pip-cache
-        run: |
-          echo "::set-output name=dir::$(pip cache dir)"
-
-      - name: Cache pip
+      - name: Cache uv
        uses: actions/cache@v3
-        id: cache-pip
        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-venv-
+            ${{ runner.os }}-${{ matrix.python }}-uv-

      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

-      - name: Install pip dependencies
+      - name: Install dependencies with uv
        run: |
-          python -m pip install --upgrade pip setuptools wheel pytest bottle build
-          python -m pip install -r requirements.txt
-          python -m pip install -e .[sonic,ldap]
-
-      - name: Get npm cache dir
-        id: npm-cache
-        run: |
-          echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
-
-      - name: Cache npm
-        uses: actions/cache@v3
-        id: cache-npm
-        with:
-          path: ${{ steps.npm-cache.outputs.dir }}
-          key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node_modules
-
-      - name: Install npm requirements
-        run: |
-          npm install
-          echo "SINGLEFILE_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/single-file" >> $GITHUB_ENV
-          echo "READABILITY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor" >> $GITHUB_ENV
-          echo "MERCURY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/mercury-parser" >> $GITHUB_ENV
+          uv sync --dev --all-extras --no-sources --no-cache
+          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl

      ### Run the tests
      - name: Directory listing for debugging
@@ -96,13 +79,24 @@ jobs:

      - name: Archivebox version
        run: |
-          archivebox version
+          mkdir -p tests/out/data
+          DATA_DIR="$PWD/tests/out/data" uv run --no-sync --no-sources archivebox version

      - name: Test built package with pytest
        # TODO: remove this exception for windows once we get tests passing on that platform
        if: ${{ !contains(matrix.os, 'windows') }}
        run: |
-          python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist
+          mkdir -p tests/out
+          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs
+
+      - name: Run plugin tests
+        if: ${{ !contains(matrix.os, 'windows') }}
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
+        run: |
+          uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage

  docker_tests:
    runs-on: ubuntu-latest
@@ -1,11 +1,21 @@
 .DS_Store
 ._*
+/.heartbeat.json
+/package.json
+/package-lock.json
 *.pyc
 __pycache__/
 .mypy_cache/
 .eggs/
 tests/out/

+# Coverage
+.coverage
+.coverage.*
+coverage.json
+coverage/
+htmlcov/
+
 # Python and Node dependencies
 venv/
 .venv/
@@ -18,6 +28,7 @@ pdm.dev.lock
 requirements-dev.txt

 # Packaging artifacts
+requirements.txt
 .pdm-python
 .pdm-build
 archivebox.egg-info
@@ -27,13 +38,27 @@ dist/

 # Data folders
 lib/
+out/
 tmp/
 data/
 data*/
+archivebox/tests/data/
+archive/
 output/
+logs/
+/-
+/personas/
+/sources/
 index.sqlite3
+queue.sqlite3
 *.sqlite*
 data.*
+.archivebox_id
+ArchiveBox.conf
+*.stdout
+*.stderr
+*.log
+.tmp/

 # vim
 *.sw?
@@ -1,9 +0,0 @@
-[submodule "docs"]
-    path = docs
-    url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
-[submodule "archivebox/vendor/pocket"]
-	path = archivebox/vendor/pocket
-	url = https://github.com/tapanpandita/pocket
-[submodule "archivebox/vendor/pydantic-pkgr"]
-	path = archivebox/vendor/pydantic-pkgr
-	url = https://github.com/ArchiveBox/pydantic-pkgr
@@ -0,0 +1,67 @@
+default_language_version:
+  python: python3.13
+
+repos:
+  - repo: https://github.com/asottile/yesqa
+    rev: v1.5.0
+    hooks:
+      - id: yesqa
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell
+        additional_dependencies:
+          - tomli
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.20.0
+    hooks:
+      - id: pyupgrade
+        args: [--py313-plus]
+
+  - repo: https://github.com/asottile/add-trailing-comma
+    rev: v3.1.0
+    hooks:
+      - id: add-trailing-comma
+
+  - repo: local
+    hooks:
+      - id: ruff-format
+        name: ruff-format
+        entry: uv run --active --no-sources ruff format
+        language: system
+        types_or: [python, pyi]
+      - id: ruff-check
+        name: ruff-check
+        entry: uv run --active --no-sources ruff check --fix
+        language: system
+        types_or: [python, pyi]
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-ast
+      - id: check-toml
+      - id: check-yaml
+        exclude: ^\.github/workflows/homebrew\.yml$
+      - id: check-json
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: destroyed-symlinks
+      - id: check-case-conflict
+      - id: check-illegal-windows-names
+      - id: check-shebang-scripts-are-executable
+        exclude: ^(archivebox/.*\.py|archivebox/tests/.*\.py|archivebox/personas/export_browser_state\.js)$
+      - id: mixed-line-ending
+      - id: fix-byte-order-marker
+      - id: end-of-file-fixer
+      - id: detect-private-key
+      - id: debug-statements
+      - id: forbid-submodules
+        exclude: ^docs$
+      - id: check-added-large-files
+        args: ["--maxkb=600"]
+      - id: name-tests-test
+        args: ["--pytest-test-first"]
+        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)
@@ -1,33 +0,0 @@
-# Read the Docs configuration file for Sphinx projects
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-version: 2
-
-submodules:
-  include: all
-  recursive: true
-
-# Set the OS, Python version and other tools you might need
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.12"
-    nodejs: "20"
-
-# Build documentation in the "docs/" directory with Sphinx
-sphinx:
-  configuration: docs/conf.py
-  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
-  # builder: "dirhtml"
-
-# Optionally build your docs in additional formats such as PDF and ePub
-formats:
-  - pdf
-  - epub
-
-# Optional but recommended, declare the Python requirements required
-# to build your documentation
-# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-python:
-  install:
-    - requirements: docs/requirements.txt
@@ -0,0 +1,497 @@
+# Claude Code Development Guide for ArchiveBox
+
+## Quick Start
+
+```bash
+# Set up dev environment (always use uv, never pip directly)
+uv sync --dev --all-extras
+
+# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
+```
+
+## Development Environment Setup
+
+### Prerequisites
+- Python 3.11+ (3.13 recommended)
+- uv package manager
+- A non-root user for running tests (e.g., `testuser`)
+
+### Install Dependencies
+```bash
+uv sync --dev --all-extras  # Always use uv, never pip directly
+```
+
+### Activate Virtual Environment
+```bash
+source .venv/bin/activate
+```
+
+### Common Gotchas
+
+#### File Permissions
+New files created by root need permissions fixed for testuser:
+```bash
+chmod 644 archivebox/tests/test_*.py
+```
+
+#### DATA_DIR Environment Variable
+ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
+
+## Code Style Guidelines
+
+### Naming Conventions for Grep-ability
+Use consistent naming for everything to enable easy grep-ability and logical grouping:
+
+**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
+
+**Examples**:
+```python
+# Filesystem migration methods - all start with fs_
+def fs_migration_needed() -> bool: ...
+def fs_migrate() -> None: ...
+def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
+def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
+def _fs_next_version(current: str) -> str: ...
+
+# Logging methods - ALL must start with log_ or _log
+def log_migration_start(snapshot_id: str) -> None: ...
+def _log_error(message: str) -> None: ...
+def log_validation_result(ok: bool, msg: str) -> None: ...
+```
+
+**Rules**:
+- Group related functions with common prefixes
+- Use `_` prefix for internal/private helpers within the same family
+- ALL logging-related methods MUST start with `log_` or `_log`
+- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
+- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
+
+### Minimize Unique Names and Data Structures
+**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
+
+**Example - GOOD**:
+```python
+# Binary has overrides field
+binary = Binary(overrides={'TIMEOUT': '60s'})
+
+# Binary reuses the same field name and structure
+class Binary(models.Model):
+    overrides = models.JSONField(default=dict)  # Same name, same structure
+```
+
+**Example - BAD**:
+```python
+# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
+class Binary(models.Model):
+    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
+```
+
+**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
+
+## Testing
+
+### CRITICAL: Never Run as Root
+ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
+
+```bash
+# Run all migration tests
+sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
+
+# Run specific test file
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
+
+# Run single test
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
+```
+
+### Test File Structure
+```
+archivebox/tests/
+├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
+├── test_migrations_fresh.py      # Fresh install tests
+├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
+├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
+└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
+```
+
+### Test Writing Standards
+
+#### NO MOCKS - Real Tests Only
+Tests must exercise real code paths:
+- Create real SQLite databases with version-specific schemas
+- Seed with realistic test data
+- Run actual `python -m archivebox` commands via subprocess
+- Query SQLite directly to verify results
+
+**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
+
+#### NO SKIPS
+Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
+
+#### Strict Assertions
+- `init` command must return exit code 0 (not `[0, 1]`)
+- Verify ALL data is preserved, not just "at least one"
+- Use exact counts (`==`) not loose bounds (`>=`)
+
+### Example Test Pattern
+```python
+def test_migration_preserves_snapshots(self):
+    """Migration should preserve all snapshots."""
+    result = run_archivebox(self.work_dir, ['init'], timeout=45)
+    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
+
+    ok, msg = verify_snapshot_count(self.db_path, expected_count)
+    self.assertTrue(ok, msg)
+```
+
+### Testing Gotchas
+
+#### Extractors Disabled for Speed
+Tests disable all extractors via environment variables for faster execution:
+```python
+env['SAVE_TITLE'] = 'False'
+env['SAVE_FAVICON'] = 'False'
+# ... etc
+```
+
+#### Timeout Settings
+Use appropriate timeouts for migration tests (45s for init, 60s default).
+
+### Plugin Testing & Code Coverage
+
+**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
+
+```bash
+# Run plugin tests with coverage (both Python + JavaScript)
+bash bin/test_plugins.sh screenshot
+
+# View coverage reports
+bash bin/test_plugins.sh --coverage-report
+# Or individual reports:
+coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
+```
+
+#### Plugin Test Structure
+
+Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
+
+```python
+# Correct production paths:
+# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
+# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
+
+with tempfile.TemporaryDirectory() as tmpdir:
+    data_dir = Path(tmpdir)
+
+    # Crawl-level plugin (e.g., chrome launcher)
+    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
+    chrome_dir = crawl_dir / 'chrome'
+    chrome_dir.mkdir(parents=True)
+
+    # Snapshot-level plugin (e.g., screenshot)
+    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
+    screenshot_dir = snapshot_dir / 'screenshot'
+    screenshot_dir.mkdir(parents=True)
+
+    # Run hook in its output directory
+    result = subprocess.run(
+        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
+        cwd=str(screenshot_dir),
+        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
+        capture_output=True,
+        timeout=120
+    )
+```
+
+#### Coverage Improvement Loop
+
+To improve from ~20% to 80%+:
+
+1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
+2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
+3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
+4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
+
+**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
+
+## Database Migrations
+
+### Generate and Apply Migrations
+```bash
+# Generate migrations (run from archivebox subdirectory)
+cd archivebox
+./manage.py makemigrations
+
+# Apply migrations to test database
+cd data/
+archivebox init
+```
+
+### Schema Versions
+- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
+- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
+- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
+- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
+
+### Testing a Migration Path
+1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
+2. Seed with realistic test data using `seed_0_X_data()`
+3. Run `archivebox init` to trigger migrations
+4. Verify data preservation with `verify_*` functions
+5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
+
+### Squashed Migrations
+When testing 0.8.x (dev branch), you must record ALL replaced migrations:
+```python
+# The squashed migration replaces these - all must be recorded
+('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
+('core', '0024_auto_20240513_1143'),
+# ... all 52 migrations from 0023-0074 ...
+('core', '0023_new_schema'),  # Also record the squashed migration itself
+```
+
+### Migration Strategy
+- Squashed migrations for clean installs
+- Individual migrations recorded for upgrades from dev branch
+- `replaces` attribute in squashed migrations lists what they replace
+
+### Migration Gotchas
+
+#### Circular FK References in Schemas
+SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
+
+## Plugin System Architecture
+
+### Plugin Dependency Rules
+
+Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
+However, they are allowed to depend on two shared files ONLY:
+- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
+- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
+
+### Chrome-Dependent Plugins
+
+Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
+
+**Main puppeteer-based chrome installer + launcher plugin**:
+- `chrome` - Core Chrome integration (CDP, launch, navigation)
+
+**Metadata extraction using chrome/chrome_utils.js / CDP**:
+- `dns` - DNS resolution info
+- `ssl` - SSL certificate info
+- `headers` - HTTP response headers
+- `redirects` - Capture redirect chains
+- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
+- `responses` - Capture network responses
+- `consolelog` - Capture console.log output
+- `title` - Extract page title
+- `accessibility` - Extract accessibility tree
+- `seo` - Extract SEO metadata
+
+**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
+- `ublock` - uBlock Origin ad blocking
+- `istilldontcareaboutcookies` - Cookie banner dismissal
+- `twocaptcha` - 2captcha CAPTCHA solver integration
+
+**Page-alteration plugins to prepare the content for archiving**:
+- `modalcloser` - Modal dialog dismissal
+- `infiniscroll` - Infinite scroll handler
+
+**Main Extractor Outputs**:
+- `dom` - DOM snapshot extraction
+- `pdf` - Generate PDF snapshots
+- `screenshot` - Generate screenshots
+- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
+
+**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
+- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
+- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
+- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
+- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
+
+### Finding Chrome-Dependent Plugins
+
+```bash
+# Find all files containing "chrom" (case-insensitive)
+grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
+
+# Or get just the plugin names
+grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
+```
+
+**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
+
+## Architecture Notes
+
+### Crawl Model (0.9.x)
+- Crawl groups multiple Snapshots from a single `add` command
+- Each `add` creates one Crawl with one or more Snapshots
+- Seed model was removed - crawls now store URLs directly
+
+## Code Coverage
+
+### Overview
+
+Coverage tracking is enabled for passive collection across all contexts:
+- Unit tests (pytest)
+- Integration tests
+- Dev server (manual testing)
+- CLI usage
+
+Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
+
+### Install Coverage Tools
+
+```bash
+uv sync --dev  # Installs pytest-cov and coverage
+```
+
+### Running with Coverage
+
+#### Unit Tests
+```bash
+# Run tests with coverage
+pytest --cov=archivebox --cov-report=term archivebox/tests/
+
+# Or run specific test file
+pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
+```
+
+#### Dev Server with Coverage
+```bash
+# Start dev server with coverage tracking
+coverage run --parallel-mode -m archivebox server
+
+# Or CLI commands
+coverage run --parallel-mode -m archivebox init
+coverage run --parallel-mode -m archivebox add https://example.com
+```
+
+#### Manual Testing (Always-On)
+To enable coverage during ALL Python executions (passive tracking):
+
+```bash
+# Option 1: Use coverage run wrapper
+coverage run --parallel-mode -m archivebox [command]
+
+# Option 2: Set environment variable (tracks everything)
+export COVERAGE_PROCESS_START=pyproject.toml
+# Now all Python processes will track coverage
+archivebox server
+archivebox add https://example.com
+```
+
+### Viewing Coverage
+
+#### Text Report (Quick View)
+```bash
+# Combine all parallel coverage data
+coverage combine
+
+# View summary
+coverage report
+
+# View detailed report with missing lines
+coverage report --show-missing
+
+# View specific file
+coverage report --include="archivebox/core/models.py" --show-missing
+```
+
+#### JSON Report (LLM-Friendly)
+```bash
+# Generate JSON report
+coverage json
+
+# View the JSON
+cat coverage.json | jq '.files | keys'  # List all files
+
+# Find files with low coverage
+cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
+
+# Find completely uncovered files (dead code candidates)
+cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
+
+# Get missing lines for a specific file
+cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
+```
+
+#### HTML Report (Visual)
+```bash
+# Generate interactive HTML report
+coverage html
+
+# Open in browser
+open htmlcov/index.html
+```
+
+### Isolated Runs
+
+To measure coverage for specific scenarios:
+
+```bash
+# 1. Reset coverage data
+coverage erase
+
+# 2. Run your isolated test/scenario
+pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
+# OR
+coverage run --parallel-mode -m archivebox add https://example.com
+
+# 3. View results
+coverage combine
+coverage report --show-missing
+
+# 4. Optionally export for analysis
+coverage json
+```
+
+### Finding Dead Code
+
+```bash
+# 1. Run comprehensive tests + manual testing to build coverage
+pytest --cov=archivebox archivebox/tests/
+coverage run --parallel-mode -m archivebox server  # Use the app manually
+coverage combine
+
+# 2. Find files with 0% coverage (strong dead code candidates)
+coverage json
+cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
+
+# 3. Find files with <10% coverage (likely dead code)
+cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
+
+# 4. Generate detailed report for analysis
+coverage report --show-missing > coverage_report.txt
+```
+
+### Tips
+
+- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
+- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
+- **Reset**: Use `coverage erase` to start fresh for isolated measurements
+- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
+- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
+
+## Debugging Tips
+
+### Check Migration State
+```bash
+sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
+```
+
+### Check Table Schema
+```bash
+sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
+```
+
+### Verbose Test Output
+```bash
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
+```
+
+### Kill Zombie Chrome Processes
+```bash
+./bin/kill_chrome.sh
+```
@@ -1 +0,0 @@
-archivebox.io
@@ -1,24 +1,38 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
-#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file
+# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
+#     python3.13, uv, python3-ldap
+#     curl, wget, git, dig, ping, tree, nano
+#     node, npm
+#     ArchiveBox and plugin runtime dependencies installed by archivebox init --install
 # Usage:
-#     git submodule update --init --recursive
-#     git pull --recurse-submodules
-#     docker build . -t archivebox --no-cache
+#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
+#     docker build . -t archivebox
 #     docker run -v "$PWD/data":/data archivebox init
 #     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
 #     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
-# Multi-arch build:
-#     docker buildx create --use
-#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:0.7.5 -t archivebox/archivebox:dev
-#
-# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
+#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
+# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+

 #########################################################################################

-# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
-FROM python:3.11-slim-bookworm
-# FROM debian:bookworm-backports
+### Example: Using ArchiveBox in your own project's Dockerfile ########
+
+# FROM python:3.13-slim
+# WORKDIR /data
+# RUN pip install archivebox>=0.9.0   # use latest release here
+# RUN archivebox install
+# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
+
+#########################################################################################
+
+ARG TARGETPLATFORM
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT=
+
+FROM archivebox/sonic:1.4.9 AS sonic
+FROM ubuntu:24.04

 LABEL name="archivebox" \
    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -44,10 +58,9 @@ ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
-
 ######### Environment Variables #################################

-# Global system-level config
+# Global build-time and runtime environment constants + default pkg manager config
 ENV TZ=UTC \
    LANGUAGE=en_US:en \
    LC_ALL=C.UTF-8 \
@@ -59,40 +72,41 @@ ENV TZ=UTC \
    PIP_DISABLE_PIP_VERSION_CHECK=1 \
    npm_config_loglevel=error

-# Version config
-ENV PYTHON_VERSION=3.11 \
-    NODE_VERSION=22
+# Language Version config
+ENV PYTHON_VERSION=3.13 \
+    NODE_VERSION=22.22.3

-# User config
+# Non-root User config
 ENV ARCHIVEBOX_USER="archivebox" \
    DEFAULT_PUID=911 \
    DEFAULT_PGID=911 \
    IN_DOCKER=True

-# Global paths
+# ArchiveBox Source Code + Lib + Data paths
 ENV CODE_DIR=/app \
    DATA_DIR=/data \
+    LIB_DIR=/opt/archivebox/lib \
+    LIB_BIN_DIR=/opt/archivebox/lib/bin \
+    ABXPKG_LIB_DIR=/opt/archivebox/lib \
    PLAYWRIGHT_BROWSERS_PATH=/browsers
-    # GLOBAL_VENV=/venv \
-    # TODO: add TMP_DIR and LIB_DIR?

-# Build shell config
+# Bash SHELL config
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 

 ######### System Environment ####################################

-# Detect ArchiveBox version number by reading pyproject.toml
-COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN grep '^version = ' "${CODE_DIR}/pyproject.toml" | awk -F'"' '{print $2}' > /VERSION.txt
+# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
+WORKDIR "$CODE_DIR"

-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
+# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
 RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
    && rm -f /etc/apt/apt.conf.d/docker-clean

 # Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
+RUN (echo "[i] Docker build for ArchiveBox starting..." \
    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
    && echo \
@@ -100,9 +114,9 @@ RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
    && echo \
    && uname -a \
-    && cat /etc/os-release | head -n7 \
-    && which bash && bash --version | head -n1 \
-    && which dpkg && dpkg --version | head -n1 \
+    && sed -n '1,7p' /etc/os-release \
+    && which bash && bash --version | sed -n '1p' \
+    && which dpkg && dpkg --version | sed -n '1p' \
    && echo -e '\n\n' && env && echo -e '\n\n' \
    ) | tee -a /VERSION.txt

@@ -114,38 +128,38 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
+    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
    # https://docs.linuxserver.io/general/understanding-puid-and-pgid

 # Install system apt dependencies (adding backports to access more recent apt updates)
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
-    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' > /etc/apt/sources.list.d/backports.list \
+    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
    && mkdir -p /etc/apt/keyrings \
    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
+    && apt-get install -qq -y \
        # 1. packaging dependencies
        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils \
+        zlib1g-dev dumb-init gosu cron unzip grep dnsutils git python3.12-venv \
        # 3. frivolous CLI helpers to make debugging failed archiving easier
        tree nano iputils-ping \
        # nano iputils-ping dnsutils htop procps jq yq
    && rm -rf /var/lib/apt/lists/*

 # Install sonic search backend
-COPY --from=archivebox/sonic:1.4.9 /usr/local/bin/sonic /usr/local/bin/sonic
+COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
 RUN (which sonic && sonic --version) | tee -a /VERSION.txt

 ######### Language Environments ####################################

-# Install Python environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
-    # NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
+# Set up Python environment
+# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
+#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
    # && apt-get update -qq \
-    # && apt-get install -qq -y -t bookworm-backports --no-upgrade \
+    # && apt-get install -qq -y --no-upgrade \
    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
    # && rm -rf /var/lib/apt/lists/* \
    # tell PDM to allow using global system python site packages
@@ -158,26 +172,30 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    # install global dependencies / python build dependencies in GLOBAL_VENV
    # && pip install --upgrade pip setuptools wheel \
    # Save version info
-    && ( \
-        which python3 && python3 --version | grep " $PYTHON_VERSION" \
-        && which pip && pip --version \
-        # && which pdm && pdm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
+    # && ( \
+    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
+    #     && which pip && pip --version \
+    #     # && which pdm && pdm --version \
+    #     && echo -e '\n\n' \
+    # ) | tee -a /VERSION.txt


-# Install Node environment
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Node $NODE_VERSION environment..." \
-    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
-    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports --no-upgrade libatomic1 \
-    && apt-get install -y -t bookworm-backports --no-upgrade \
-        nodejs \
-    && rm -rf /var/lib/apt/lists/* \
-    # Update NPM to latest version
-    && npm i -g npm --cache /root/.npm \
+# Set up Node environment from the official platform tarball. This avoids
+# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
+# emulated Docker builds.
+RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    case "$TARGETARCH" in \
+        amd64) NODE_DIST_ARCH="x64" ;; \
+        arm64) NODE_DIST_ARCH="arm64" ;; \
+        *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \
+    esac \
+    && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \
+    && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \
+    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \
+    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \
+    && grep "  ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \
+    && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \
+    && rm "$NODE_TARBALL" SHASUMS256.txt \
    # Save version info
    && ( \
        which node && node --version \
@@ -186,141 +204,192 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    ) | tee -a /VERSION.txt


-
-######### Extractor Dependencies ##################################
-
-# Install apt dependencies
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing APT extractor dependencies globally using apt..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        curl wget git ffmpeg ripgrep \
-        # Packages we have also needed in the past:
-        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
-    && rm -rf /var/lib/apt/lists/* \
-    # Save version info
-    && ( \
-        which curl && curl --version | head -n1 \
-        && which wget && wget --version 2>&1 | head -n1 \
-        && which git && git --version 2>&1 | head -n1 \
-        && which rg && rg --version 2>&1 | head -n1 \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-# Install chromium browser using playwright
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
-        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-        at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
-        libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
-        libxaw7 libxcomposite1 libxdamage1 libxfont2 \
-        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
-        # xfonts-scalable xfonts-utils xserver-common xvfb \
-        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
-        # libxss1 dbus dbus-x11 upower \
-    # && service dbus start \
-    # install Chromium using playwright
-    && pip install playwright \
-    && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
-    && playwright install chromium \
-    && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
-    && rm -rf /var/lib/apt/lists/* \
-    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
-    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \
-    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
-    # Save version info
-    && ( \
-        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Install Node dependencies
-ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH"
-USER $ARCHIVEBOX_USER
-WORKDIR "/home/$ARCHIVEBOX_USER/.npm"
-RUN --mount=type=cache,target=/home/$ARCHIVEBOX_USER/.npm_cache,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT,uid=$DEFAULT_PUID,gid=$DEFAULT_PGID \
-    echo "[+] Installing NPM extractor dependencies in /home/$ARCHIVEBOX_USER/.npm..." \
-    && npm config set prefix "/home/$ARCHIVEBOX_USER/.npm" \
-    && npm install --global --prefer-offline --no-fund --no-audit --cache "/home/$ARCHIVEBOX_USER/.npm_cache" \
-        "@postlight/parser@^2.2.3" \
-        "readability-extractor@github:ArchiveBox/readability-extractor" \
-        "single-file-cli@^1.1.54" \
-        "puppeteer@^23.5.0" \
-        "@puppeteer/browsers@^2.4.0"
-USER root
-RUN ( \
-        which node && node --version \
-        && which npm && npm version \
-        && which postlight-parser \
-        && which readability-extractor && readability-extractor --version \
-        && which single-file && single-file --version \
-        && which puppeteer && puppeteer --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-######### Build Dependencies ####################################
-
-# Install ArchiveBox Python dependencies
+# Set up uv and main app /venv
+RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
+ENV UV_COMPILE_BYTECODE=0 \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv
 WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
+# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
+    && uv venv /venv --python ${PYTHON_VERSION}
+ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
+RUN uv pip install setuptools pip \
+    && ( \
+        which python3 && python3 --version \
+        && which uv && uv self version \
+        && uv python find --system && uv python find \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+
+######### ArchiveBox & Extractor Dependencies ##################################
+
+# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
    && apt-get update -qq \
-    && apt-get install -qq -y -t bookworm-backports \
+    && apt-get install -qq -y --no-install-recommends \
        build-essential gcc \
-        libssl-dev libldap2-dev libsasl2-dev \
-        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
-    # && pdm use --venv in-project \
-    # && pdm run python -m ensurepip \
-    # && pdm sync --fail-fast --no-editable --group :all --no-self \
-    # && pdm export -o requirements.txt --without-hashes \
-    # && source $GLOBAL_VENV/bin/activate \
-    && pip install -r requirements.txt \
+        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
+        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
+    && uv pip install \
+        "python-ldap>=3.4.3" \
    && apt-get purge -y \
-        build-essential gcc \
+        python3-dev build-essential gcc \
    && apt-get autoremove -y \
    && rm -rf /var/lib/apt/lists/*

-# Install ArchiveBox Python package from source
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
-    && pip install -e "${CODE_DIR}[all]" \
-    && rm -rf /var/lib/apt/lists/*

-####################################################
+# Runtime config used by plugin hooks. Plugin binaries and npm packages are
+# installed into LIB_DIR below by archivebox init --install and resolved from
+# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
+ENV PERSONAS_DIR=/data/personas \
+    CHROME_EXTENSIONS_DIR=/opt/archivebox/lib/chrome_extensions \
+    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
+    CHROME_HEADLESS=true \
+    CHROME_SANDBOX=false \
+    CHROME_ISOLATION=crawl \
+    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+
+######### Build Dependencies ####################################
+
+
+# Install ArchiveBox Python venv dependencies from pyproject.toml.
+RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \
+    && uv --no-cache sync \
+        --refresh \
+        --no-dev \
+        --inexact \
+        --all-extras \
+        --no-install-project \
+        --no-install-workspace \
+        --no-sources \
+    && apt-get purge -y python3-dev build-essential gcc \
+    && apt-get autoremove -y \
+    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && rm -rf /var/lib/apt/lists/*
+    # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies

 # Setup ArchiveBox runtime config
+ENV TMP_DIR=/tmp/archivebox \
+    PIP_VENV_PYTHON=/usr/bin/python3.12 \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
 WORKDIR "$DATA_DIR"
 RUN openssl rand -hex 16 > /etc/machine-id \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/tmp"
-ENV GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no \
-    ALLOWED_HOSTS=*
+    && mkdir -p "$DATA_DIR" \
+    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \
+    && mkdir -p "$TMP_DIR" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
+    && mkdir -p "$LIB_DIR" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
+    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
+    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+
+# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer
+# path users run later, before copying ArchiveBox source so source-only edits do
+# not invalidate the heavy browser/plugin dependency layer.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/opt/archivebox/lib,sharing=locked,id=archivebox-lib-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
+    && export PERSONAS_DIR="$LIB_DIR/personas" \
+    && export CHROME_EXTENSIONS_DIR="$LIB_DIR/chrome_extensions" \
+    && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
+    && mkdir -p "$LIB_DIR" "$LIB_DIR/chrome_extensions" \
+    && apt-get update -qq \
+    && if [ "$TARGETARCH" = "arm64" ]; then \
+        abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
+        abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
+    fi \
+    && TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \
+    && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \
+    && mkdir -p /opt/archivebox/lib-layer \
+    && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \
+    && rm -rf /var/lib/apt/lists/* \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer
+
+RUN rm -rf "$LIB_DIR" \
+    && mv /opt/archivebox/lib-layer "$LIB_DIR" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
+
+# Install ArchiveBox Python package from the checked-out source.
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && pip install \
+        --no-deps \
+        "$CODE_DIR" \
+    && ( \
+        pip show archivebox \
+        && which archivebox \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt \
+    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
+    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
+
+# Initialize an empty image collection without rerunning dependency installs.
+WORKDIR "$DATA_DIR"
+RUN echo "[+] Initializing image collection..." \
+    && PUID=0 PGID=0 archivebox init \
+    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
+    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
+        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
+        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
+        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
+        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
+        2>/dev/null || true)

 # Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
+RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
    ) | tee -a /VERSION.txt
-RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
+
+# Verify ArchiveBox is installed and write full version/dependency info.
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
+    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
+    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
+    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
+    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*

 ####################################################

-# Open up the interfaces to the outside world
+# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'
+    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'

 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]
+CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]
@@ -1,6 +1,6 @@
 MIT License

-Copyright (c) 2020 Nick Sweeting
+Copyright (c) 2024 Nick Sweeting

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -13,23 +13,25 @@
 <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE"><img src="https://img.shields.io/badge/Open_source-MIT-green.svg?logo=git&logoColor=green"/></a> <a href="https://github.com/ArchiveBox/ArchiveBox/commits/dev"><img src="https://img.shields.io/github/last-commit/ArchiveBox/ArchiveBox.svg?logo=Sublime+Text&logoColor=green&label=Active"/></a> &nbsp; <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?logo=github&label=Stars&logoColor=blue"/></a> &nbsp; <a href="https://hub.docker.com/r/archivebox/archivebox"><img src="https://img.shields.io/docker/pulls/archivebox/archivebox.svg?label=Docker+Pulls"/></a> <a href="https://pypi.org/project/archivebox/"><img src="https://img.shields.io/pypi/dm/archivebox?label=PyPI%20Installs&color=%235f7dae"/></a> <a href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj"><img src="https://img.shields.io/chrome-web-store/users/habonpimjphpdnmcfkaockjnffodikoj?label=Chrome%20Store&color=%231973e8"/></a>

 <!--<pre lang="bash" align="left"><code style="white-space: pre-line; text-align: left" align="left">
-curl -fsSL 'https://get.archivebox.io' | sh    # (or see pip/brew/Docker instructions below)
+curl -fsSL 'https://get.archivebox.io' | bash    # (or see pip/brew/Docker instructions below)
 </code></pre>-->

 </div>
 <hr/>
 <br/>

-**ArchiveBox is a powerful, self-hosted internet archiving solution to collect, save, and view websites offline.**
+**ArchiveBox is a self-hosted app that lets you preserve content from websites in a variety of formats.**

-Without active preservation effort, everything on the internet eventually disappears or degrades. Archive.org does a great job as a centralized service, but saved URLs have to be public, and they can't save every type of content.
+We aim to make your data immediately useful, and kept in formats that other programs can read directly. As output, we save standard HTML, PNG, PDF, TXT, JSON, WARC, SQLite, all guaranteed to be readable for decades to come. ArchiveBox also has a CLI, REST API, and webhooks so you can set up integrations with other services.
+
+Without active preservation effort, everything on the internet eventually disappears or degrades.

 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>

-> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart), [macOS](#quickstart), and [Windows](#quickstart) (WSL2), or via **[Docker](#quickstart)** ⭐️.  
+> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  

-*Once installed, it can be used as a [CLI tool](#usage), [self-hosted Web App](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python library](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [one-off command](#static-archive-exporting).*
+*Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*

 <br/>
 <hr/>
@@ -75,7 +77,7 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 <pre lang="bash"><code style="white-space: pre-line"># Option A: Get ArchiveBox with Docker Compose (recommended):
 mkdir -p ~/archivebox/data && cd ~/archivebox
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml   # edit options in this file as-needed
-docker compose run archivebox init --setup
+docker compose run archivebox init
 # docker compose run archivebox add 'https://example.com'
 # docker compose run archivebox help
 # docker compose up
@@ -83,7 +85,7 @@ docker compose run archivebox init --setup
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -it -v $PWD:/data archivebox/archivebox init --setup
+docker run -it -v $PWD:/data archivebox/archivebox init
 # docker run -it -v $PWD:/data archivebox/archivebox add 'https://example.com'
 # docker run -it -v $PWD:/data archivebox/archivebox help
 # docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox
@@ -92,17 +94,19 @@ docker run -it -v $PWD:/data archivebox/archivebox init --setup
 # Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
 pip install archivebox
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --setup
+archivebox init
+archivebox install
 # archivebox add 'https://example.com'
 # archivebox help
 # archivebox server 0.0.0.0:8000
 <br/>
 <br/>
 # Option D: Or use the optional auto setup script to install it
-curl -fsSL 'https://get.archivebox.io' | sh
+curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
-<sub>Open <a href="http://localhost:8000"><code>http://localhost:8000</code></a> to see your server's Web UI ➡️</sub>
+<sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
+<sub>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
 </details>
 <br/>

@@ -130,7 +134,7 @@ curl -fsSL 'https://get.archivebox.io' | sh
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
 - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
 - Advanced users: support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
@@ -150,9 +154,9 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 - **Individuals:**
  `saving bookmarks`, `preserving portfolio content`, `legacy / memoirs archival`
 - **Governments:**
-  `snapshoting public service sites`, `recordkeeping compliance`
+  `snapshotting public service sites`, `recordkeeping compliance`

-> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  (we are also seeking [grant funding](https://github.com/ArchiveBox/ArchiveBox/issues/1126#issuecomment-1487431394))  
+> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  
 > We offer: setup & support, CAPTCHA/ratelimit unblocking, SSO, audit logging/chain-of-custody, and more  
 > *ArchiveBox is a 🏛️ 501(c)(3) [nonprofit FSP](https://hackclub.com/hcb/) and all our work supports open-source development.* 

@@ -186,9 +190,9 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
 </code></pre></li>
 <li>Run the initial setup to create an admin user (or set ADMIN_USER/PASS in docker-compose.yml)
-<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init --setup
+<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init
 </code></pre></li>
-<li>Next steps: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
+<li>Next steps: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">docker compose up
 # completely optional, CLI can always be used without running a server
 # docker compose run [-T] archivebox [subcommand] [--help]
@@ -210,10 +214,10 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -v $PWD:/data -it archivebox/archivebox init --setup
+docker run -v $PWD:/data -it archivebox/archivebox init
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox
 # completely optional, CLI can always be used without running a server
 # docker run -v $PWD:/data -it [subcommand] [--help]
@@ -233,7 +237,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (optional, highly recommended but not required).</li>
 <li>Run the automatic setup script.
-<pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | sh</code></pre>
+<pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>
@@ -256,23 +260,22 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 <br/>
 <ol>

-<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.10</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v18</a> on your system (if not already installed).</li>
-<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://pipx.pypa.io"><code>pipx</code></a>).
-<pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox yt-dlp playwright
-playwright install --with-deps chromium
+<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.13</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v22</a> on your system (if not already installed).</li>
+<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://docs.astral.sh/uv/guides/tools/#running-tools"><code>uvx</code></a>).
+<pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox
 archivebox version
 # install any missing extras shown using apt/brew/pkg/etc. see Wiki for instructions
-#    python@3.10 node curl wget git ripgrep ...
+#    python@3.13 node curl wget git ripgrep ...
 </code></pre>
 <i>See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install">Install: Bare Metal</a> Wiki for full install instructions for each OS...</i>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data   # for example
-archivebox init --setup   # instantialize a new collection
-# (--setup auto-installs and link JS dependencies: singlefile, readability, mercury, etc.)
+archivebox init     # instantialize a new collection
+archivebox install  # install all the runtime dependencies (e.g. chrome, single-file, yt-dlp, etc.)
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
@@ -291,29 +294,23 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <details>
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
-See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki for instructions. ➡️
-<!--<ol>
-<li>Add the ArchiveBox repository to your sources.<br/>
-<pre lang="bash"><code style="white-space: pre-line">echo "deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/archivebox.list
-sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C258F79DCC02E369
-sudo apt update
-</code></pre>
-</li>
-<li>Install the ArchiveBox package using <code>apt</code>.
-<pre lang="bash"><code style="white-space: pre-line">sudo apt install archivebox
-# update to newest version with pip (sometimes apt package is outdated)
-pip install --upgrade --ignore-installed archivebox yt-dlp playwright
-playwright install --with-deps chromium    # install chromium and its system dependencies
+<ol>
+<li>Download and install the <code>.deb</code> package from the <a href="https://github.com/ArchiveBox/ArchiveBox/releases">latest release</a>.
+<pre lang="bash"><code style="white-space: pre-line"># download the .deb for your architecture (amd64 or arm64)
+ARCH="$(dpkg --print-architecture)"
+VERSION="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")"
+curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${VERSION}_${ARCH}.deb" -o /tmp/archivebox.deb
+sudo apt install /tmp/archivebox.deb
 archivebox version                         # make sure all dependencies are installed
 </code></pre>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --setup
+archivebox init --install
 </code></pre>
 <br/>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
@@ -322,7 +319,7 @@ archivebox help
 </li>
 </ol>
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
-<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>-->
+<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>
 <br/><br/>
 </details>

@@ -334,19 +331,17 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install the ArchiveBox package using <code>brew</code>.
 <pre lang="bash"><code style="white-space: pre-line">brew tap archivebox/archivebox
 brew install archivebox
-# update to newest version with pip (sometimes brew package is outdated)
-pip install --upgrade --ignore-installed archivebox yt-dlp playwright
-playwright install --with-deps chromium    # install chromium and its system dependencies
 archivebox version                         # make sure all dependencies are installed
 </code></pre>
 <i>See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki for more granular instructions for macOS... ➡️</i>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --setup
+archivebox init
+archivebox install
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
@@ -367,8 +362,8 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 > *Warning: These are contributed by external volunteers and may lag behind the official `pip` channel.*

 <ul>
-<li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>)</li>
-<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | sh</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
+<li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>, maintained by <a href="https://github.com/jasongodev"><code>@jasongodev</code></a>)</li>
+<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
 <li>Nix: <a href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/archivebox/default.nix"><code>nix-env --install archivebox</code></a> (contributed by <a href="https://github.com/siraben"><code>@siraben</code></a>)</li>
 <li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
 <li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
@@ -409,6 +404,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <ul>
 <li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
 <li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
+<li><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a></li>
 <li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
 <li><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a></li>
 <li><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a></li>
@@ -429,7 +425,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <ul>
 <li><a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">
 <img src="https://img.shields.io/badge/Custom_Development-ArchiveBox.io-%231a1a1a.svg?style=flat" height="22px"/>
-</a> (<a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">get hosting, support, and feature customization directy from us</a>)</li>
+</a> (<a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">get hosting, support, and feature customization directly from us</a>)</li>
 <li><a href="https://monadical.com">
 <img src="https://img.shields.io/badge/General_Dev_Consulting-Monadical.com-%231a1a1a.svg?style=flat" height="22px"/>
 </a> (<a href="https://monadical.com/contact-us.html">generalist consultancy that has ArchiveBox experience</a>)</li>
@@ -466,6 +462,7 @@ For more discussion on managed and paid hosting options see here: <a href="https
 #### ➡️&nbsp; Next Steps

 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
+- (Optional) Create a persona and import browser cookies to archive logged-in sites: `archivebox persona create --import=chrome personal`
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -492,13 +489,19 @@ docker compose run archivebox help

 # equivalent: docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
 docker run -it -v $PWD:/data archivebox/archivebox help
+
+# optional: import your browser cookies into a persona for logged-in archiving
+archivebox persona create --import=chrome personal
+# supported: chrome/chromium/brave/edge (Chromium-based only)
+# use --profile to target a specific profile (e.g. Default, Profile 1)
+# re-running import merges/dedupes cookies.txt (by domain/path/name) but replaces chrome_user_data
 ```

 #### ArchiveBox Subcommands

 - `archivebox` `help`/`version` to see the list of available subcommands / currently installed version info
 - `archivebox` `setup`/`init`/`config`/`status`/`shell`/`manage` to administer your collection
- `archivebox` `add`/`oneshot`/`schedule` to pull in fresh URLs from [bookmarks/history/RSS/etc.](#input-formats)
+- `archivebox` `add`/`schedule` to pull in fresh URLs from [bookmarks/history/RSS/etc.](#input-formats)
 - `archivebox` `list`/`update`/`remove` to manage existing Snapshots in your collection

 <br/>
@@ -509,7 +512,7 @@ docker run -it -v $PWD:/data archivebox/archivebox help
 # make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
 <br/>
 # archivebox [subcommand] [--help]
-archivebox init --setup      # safe to run init multiple times (also how you update versions)
+archivebox init --install      # safe to run init multiple times (also how you update versions)
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
 archivebox add --depth=1 'https://news.ycombinator.com'
@@ -526,7 +529,7 @@ archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
 # docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox init --setup
+docker compose run archivebox init --install
 docker compose run archivebox version
 docker compose run archivebox help
 docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
@@ -544,7 +547,7 @@ docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you create and cd into in a new empty directory first  
 <br/>
 # docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init --setup
+docker run -v $PWD:/data -it archivebox/archivebox init --install
 docker run -v $PWD:/data -it archivebox/archivebox version
 docker run -v $PWD:/data -it archivebox/archivebox help
 docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.ycombinator.com'
@@ -584,7 +587,8 @@ docker run -v $PWD:/data -it archivebox/archivebox archivebox manage createsuper
 docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>

-<sup>Open <a href="http://localhost:8000"><code>http://localhost:8000</code></a> to see your server's Web UI ➡️</sup>
+<sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
+<sup>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
 <br/><br/>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Usage: Web UI</a> wiki. ➡️</i>
 <br/><br/>
@@ -688,7 +692,7 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a

 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.

-It also includes a built-in scheduled import feature with `archivebox schedule` and browser bookmarklet, so you can pull in URLs from RSS feeds, websites, or the filesystem regularly/on-demand.
+It also includes a built-in scheduled import feature with `archivebox schedule`, handled by the same orchestrator that powers `archivebox server`, so you can pull in URLs from RSS feeds and websites regularly without a separate cron container.

 <br/>

@@ -749,7 +753,7 @@ env CHROME_BINARY=chromium archivebox ...       # run with a one-off config
 <sub>These methods also work the same way when run inside Docker, see the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Docker Configuration</a> wiki page for details.</sub>
 </details><br/>

-The configuration is documented here: **[Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)**, and loaded here: [`archivebox/config.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config.py).
+The configuration is documented here: **[Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)**, and loaded from: [`archivebox/config/`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config/).

 <a name="most-common-options-to-tweak"></a>
 <details>
@@ -760,16 +764,12 @@ The configuration is documented here: **[Configuration Wiki](https://github.com/
 <br/>
 TIMEOUT=240                # default: 60    add more seconds on slower networks
 CHECK_SSL_VALIDITY=False   # default: True  False = allow saving URLs w/ bad SSL
-SAVE_ARCHIVE_DOT_ORG=False # default: True  False = disable Archive.org saving
-MAX_MEDIA_SIZE=1500m       # default: 750m  raise/lower youtubedl output size
 <br/>
 PUBLIC_INDEX=True          # default: True  whether anon users can view index
 PUBLIC_SNAPSHOTS=True      # default: True  whether anon users can view pages
 PUBLIC_ADD_VIEW=False      # default: False whether anon users can add new URLs
 <br/>
-CHROME_USER_AGENT="Mozilla/5.0 ..."  # change these to get around bot blocking
-WGET_USER_AGENT="Mozilla/5.0 ..."
-CURL_USER_AGENT="Mozilla/5.0 ..."
+USER_AGENT="Mozilla/5.0 ..."  # change this to get around bot blocking
 </code></pre>
 </details>
 <br/>
@@ -791,14 +791,14 @@ ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.c
 </blockquote>

 <ul>
-<li>Language: Python <code>&gt;=3.10</code></li>
+<li>Language: Python <code>&gt;=3.13</code></li>
 <li>Backend: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://django-ninja.dev/">Django-Ninja</a> for REST API</li>
-<li>Frontend: <a href="https://docs.djangoproject.com/en/5.1/ref/contrib/admin/">Django Admin</a> + Vanilla HTML, CSS, JS</li>
-<li>Web Server: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://channels.readthedocs.io/en/latest/"><code>channels</code></a> + <a href="https://github.com/django/daphne/"><code>daphne]</code></a></li>
-<li>Database: <a href="https://docs.djangoproject.com/en/5.1/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite</code></li>
-<li>Job Queue: <a href="https://huey.readthedocs.io/">Huey</a> using <code>./data/queue.sqlite3</code> under <code>supervisord</code></li>
-<li>Build/test/lint: <a href="https://github.com/pdm-project/pdm"><code>pdm</code></a> / <code>mypy</code>+<code>pyright</code>+<code>pytest</code> / <code>ruff</code></li>
-<li>Subdependencies: <a href="https://github.com/ArchiveBox/pydantic-pkgr"><code>pydantic-pkgr</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
+<li>Frontend: <a href="https://docs.djangoproject.com/en/6.0/ref/contrib/admin/">Django Admin</a> + Vanilla HTML, CSS, JS</li>
+<li>Web Server: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://github.com/django/daphne/"><code>daphne</code></a> (ASGI)</li>
+<li>Database: <a href="https://docs.djangoproject.com/en/6.0/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite3</code></li>
+<li>Job Queue: Custom orchestrator using <code>supervisord</code> for worker management</li>
+<li>Build/test/lint: <a href="https://github.com/astral-sh/uv"><code>uv</code></a> / <code>pyright</code>+<code>ty</code>+<code>pytest</code> / <code>ruff</code></li>
+<li>Subdependencies: <a href="https://github.com/ArchiveBox/abxpkg"><code>abxpkg</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
 </ul>


@@ -827,7 +827,7 @@ If not using Docker, make sure to keep the dependencies up-to-date yourself and
 # apt/brew/pip/etc install ... (see Quickstart instructions above)
 <br/>
 which -a archivebox    # see where you have installed archivebox
-archivebox setup       # auto install all the extractors and extras
+archivebox install     # auto install all the extractors and extras
 archivebox --version   # see info and check validity of installed dependencies
 </code></pre>
  
@@ -897,7 +897,7 @@ Each snapshot subfolder <code>data/archive/TIMESTAMP/</code> includes a static <

 ## Static Archive Exporting

-You can create one-off archives of individual URLs with `archivebox oneshot`, or export your index as static HTML using `archivebox list` (so you can view it without an ArchiveBox server).
+You can export your index as static HTML using `archivebox list` (so you can view it without an ArchiveBox server).

 <br/>
 <details>
@@ -907,10 +907,7 @@ You can create one-off archives of individual URLs with `archivebox oneshot`, or
 <p><em>NOTE: These exports are not paginated, exporting many URLs or the entire archive at once may be slow. Use the filtering CLI flags on the <code>archivebox list</code> command to export specific Snapshots or ranges.</em></p>
 </blockquote>

-<pre lang="bash"><code style="white-space: pre-line"># do a one-off single URL archive wihout needing a data dir initialized
-archivebox oneshot 'https://example.com'
-
-# archivebox list --help
+<pre lang="bash"><code style="white-space: pre-line"># archivebox list --help
 archivebox list --html --with-headers > index.html     # export to static html table
 archivebox list --json --with-headers > index.json     # export to json blob
 archivebox list --csv=timestamp,url,title > index.csv  # export to csv spreadsheet
@@ -955,18 +952,11 @@ If you're importing pages with private content or URLs containing secret tokens
 archivebox add 'https://docs.google.com/document/d/12345somePrivateDocument'
 archivebox add 'https://vimeo.com/somePrivateVideo'

-# without first disabling saving to Archive.org:
-archivebox config --set SAVE_ARCHIVE_DOT_ORG=False  # disable saving all URLs in Archive.org
-
 # restrict the main index, Snapshot content, and Add Page to authenticated users as-needed:
 archivebox config --set PUBLIC_INDEX=False
 archivebox config --set PUBLIC_SNAPSHOTS=False
-archivebox config --set PUBLIC_ADD_VIEW=False 
+archivebox config --set PUBLIC_ADD_VIEW=False
 archivebox manage createsuperuser
-
-# if extra paranoid or anti-Google:
-archivebox config --set SAVE_FAVICON=False          # disable favicon fetching (it calls a Google API passing the URL's domain part only)
-archivebox config --set CHROME_BINARY=chromium      # ensure it's using Chromium instead of Chrome
 </code></pre>

 <blockquote>
@@ -1089,7 +1079,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <br/>

 <ul>
-<li><strong>ArchiveBox can use anywhere from ~1gb per 1000 Snapshots, to ~50gb per 1000 Snapshots</strong>, mostly dependent on whether you're saving audio & video using <code>SAVE_MEDIA=True</code> and whether you lower <code>MEDIA_MAX_SIZE=750mb</code>.</li>
+<li><strong>ArchiveBox can use anywhere from ~1gb per 1000 Snapshots, to ~50gb per 1000 Snapshots</strong>, mostly dependent on whether you're saving video/audio using <code>YTDLP_ENABLED=True</code> and whether you lower <code>YTDLP_MAX_SIZE=750m</code>.</li>
 <li>Disk usage can be reduced by using a compressed/<a href="https://www.ixsystems.com/blog/ixsystems-and-klara-systems-celebrate-valentines-day-with-a-heartfelt-donation-of-fast-dedupe-to-openzfs-and-truenas/">deduplicated</a> filesystem like <a href="https://www.reddit.com/r/zfs/comments/t9cexx/a_simple_real_world_zfs_compression_speed_an/">ZFS</a>/BTRFS, or by turning off extractors methods you don't need. You can also deduplicate content with a tool like <a href="https://github.com/adrianlopezroche/fdupes"><code>fdupes</code></a> or <a href="https://github.com/pauldreik/rdfind"><code>rdfind</code></a>.  
 </li>
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
@@ -1195,7 +1185,7 @@ ArchiveBox's stance is that duplication of other people's content is only ethica

 In the U.S., <a href="https://guides.library.oregonstate.edu/copyright/libraries">libraries, researchers, and archivists</a> are allowed to duplicate copyrighted materials under <a href="https://libguides.ala.org/copyright/fairuse">"fair use"</a> for <a href="https://guides.cuny.edu/cunyfairuse/librarians#:~:text=One%20of%20these%20specified%20conditions,may%20be%20liable%20for%20copyright">private study, scholarship, or research</a>. Archive.org's non-profit preservation work is <a href="https://blog.archive.org/2024/03/01/fair-use-in-action-at-the-internet-archive/">covered under fair use</a> in the US, and they properly handle <a href="https://cardozoaelj.com/2015/03/20/use-of-copyright-law-to-take-down-revenge-porn/">unethical content</a>/<a href="https://help.archive.org/help/rights/">DMCA</a>/<a href="https://gdpr.eu/right-to-be-forgotten/#:~:text=An%20individual%20has%20the%20right,that%20individual%20withdraws%20their%20consent.">GDPR</a> removal requests to maintain good standing in the eyes of the law.

-As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use sofware like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal council if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and changing your instance's branding using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir"><code>CUSTOM_TEMPLATES_DIR</code></a>).
+As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use software like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal council if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and changing your instance's branding using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir"><code>CUSTOM_TEMPLATES_DIR</code></a>).

 </details>
 <br/>
@@ -1224,7 +1214,7 @@ ArchiveBox tries to be a robust, set-and-forget archiving solution suitable for

 Not all content is suitable to be archived on a centralized, publicly accessible platform. Archive.org doesn't offer the ability to save things behind login walls for good reason, as the content may not have been intended for a public audience. ArchiveBox exists to fill that gap by letting everyone save what they have access to on an individual basis, and to encourage decentralized archiving that's less succeptible to censorship or natural disasters.

-By having users store their content locally or within their organizations, we can also save much larger portions of the internet than a centralized service has the disk capcity handle. The eventual goal is to work towards federated archiving where users can share portions of their collections with each other, and with central archives on a case-by-case basis.
+By having users store their content locally or within their organizations, we can also save much larger portions of the internet than a centralized service has the disk capacity to handle. The eventual goal is to work towards federated archiving where users can share portions of their collections with each other, and with central archives on a case-by-case basis.

 <h3>Comparison With Other Self-Hosted Archiving Options</h3>

@@ -1233,7 +1223,7 @@ ArchiveBox differentiates itself from [similar self-hosted projects](https://git

 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*

-*If you want more bookmark categorization and note-taking features, check out [Archivy](https://archivy.github.io/), [Memex](https://github.com/WorldBrain/Memex), [Polar](https://getpolarized.io/), or [LinkAce](https://www.linkace.org/).*
+*If you want more bookmark categorization and note-taking features, check out [Memex](https://github.com/WorldBrain/Memex), [Hoarder](https://github.com/hoarder-app/hoarder), [LinkWarden](https://github.com/linkwarden/linkwarden),  [Archivy](https://archivy.github.io/), or [LinkAce](https://www.linkace.org/).*

 *If you need more advanced recursive spider/crawling ability beyond `--depth=1`, check out [Browsertrix](https://github.com/webrecorder/browsertrix-crawler), [Photon](https://github.com/s0md3v/Photon), or [Scrapy](https://scrapy.org/) and pipe the outputted URLs into ArchiveBox.*

@@ -1347,30 +1337,49 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc

 **Python API Documentation:** https://docs.archivebox.io/en/dev/archivebox.html#module-archivebox.main

+**Internal Architecture Diagrams:** https://github.com/ArchiveBox/ArchiveBox/wiki/ArchiveBox-Architecture-Diagrams
+
+
 ### Setup the dev environment

 <details><summary><i>Click to expand...</i></summary>

-#### 1. Clone the main code repo (making sure to pull the submodules as well)
+#### 1. Setup the monorepo
+
+First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/

 ```bash
-git clone --recurse-submodules https://github.com/ArchiveBox/ArchiveBox
-cd ArchiveBox
-git checkout dev  # or the branch you want to test
-git submodule update --init --recursive
-git pull --recurse-submodules
+git clone https://github.com/ArchiveBox/monorepo
+cd monorepo
+./bin/setup.sh
+
+# activate the monorepo venv, then cd into archivebox
+source .venv/bin/activate
+cd archivebox
 ```

+Repos included in monorepo setup:
+
+- https://github.com/ArchiveBox/abxbus
+- https://github.com/ArchiveBox/abxpkg
+- https://github.com/ArchiveBox/abx-plugins
+- https://github.com/ArchiveBox/abx-dl
+- https://github.com/ArchiveBox/ArchiveBox
+
+
 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine

 ```bash
-# Install ArchiveBox + python dependencies
-pip install uv
-uv venv
-uv sync
-
+# Install ArchiveBox runtime dependencies
+mkdir -p data && cd data
 archivebox init
-archivebox install
+archivebox install         # detect and install all extractor dependencies
+
+# Run the development server w/ autoreloading (but no bg workers)
+archivebox server --debug --reload 0.0.0.0:8000
+
+# Run the production server (with bg workers but no autoreloading)
+archivebox server 0.0.0.0:8000
 ```

 #### 2. Option B: Build the docker container and use that for development instead
@@ -1379,14 +1388,15 @@ archivebox install
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
 # inside the container will reload and pick up your changes
-docker build . -t archivebox
-docker run -it \
-    -v $PWD/data:/data \
-    archivebox init --setup
-docker run -it -p 8000:8000 \
-    -v $PWD/data:/data \
-    -v $PWD/archivebox:/app/archivebox \
-    archivebox server 0.0.0.0:8000 --debug --reload
+./bin/build_docker.sh dev
+
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev init --install
+
+# Run the development server w/ autoreloading (but no bg workers)
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
+
+# Run the production server (with bg workers but no autoreloading)
+docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server

 # (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
 # When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@@ -1404,11 +1414,15 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 <details><summary><i>Click to expand...</i></summary>

 ```bash
+# set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
+
+# OR you can run a dev server with DEBUG=True in a few ways:
+archivebox server --debug --reload 0.0.0.0:8000
 # or
-archivebox server --debug ...
-# faster dev version wo/ bg workers enabled:
-daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
+archivebox server --debug 0.0.0.0:8000
+# or
+env DEBUG=True daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
 ```

 https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
@@ -1461,7 +1475,7 @@ archivebox install
 ./bin/lint.sh
 ./bin/test.sh
 ```
-(uses `flake8`, `mypy`, and `pytest -s`)
+(uses `ruff`, `pyright`, `ty`, and `pytest -s`)

 </details>

@@ -1505,13 +1519,9 @@ archivebox manage runscript testdata
 archivebox manage reset_db

 # use django-tui to interactively explore commands
-pip install django-tui
+uv pip install django-tui
 # ensure django-tui is in INSTALLED_APPS: core/settings.py
 archivebox manage tui
-
-# show python and JS package dependency trees
-pdm list --tree
-npm ls --all
 ```

 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/dc3e9f8c-9544-46e0-a7f0-30f571b72022" width="600px" alt="ArchiveBox ORM models relatinoship graph"/>
@@ -1527,40 +1537,11 @@ npm ls --all

 #### Contributing a new extractor

-<details><summary><i>Click to expand...</i></summary>

-<br/><br/>
+Extractors are maintained in a separate repo here: https://github.com/ArchiveBox/abx-plugins (included in monorepo setup).

-ArchiveBox [`extractors`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/media.py) are external binaries or Python/Node scripts that ArchiveBox runs to archive content on a page.
+Copy a similar plugin as a template to modify, then open a new PR to add it in that repo.

-Extractors take the URL of a page to archive, write their output to the filesystem `data/archive/TIMESTAMP/EXTRACTOR/...`, and return an [`ArchiveResult`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/models.py#:~:text=return%20qs-,class%20ArchiveResult,-(models.Model)%3A) entry which is saved to the database (visible on the `Log` page in the UI).
-
-*Check out how we added **[`archivebox/extractors/singlefile.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/singlefile.py)** as an example of the process: [Issue #399](https://github.com/ArchiveBox/ArchiveBox/issues/399) + [PR #403](https://github.com/ArchiveBox/ArchiveBox/pull/403).*
-
-<br/>
-
-
-**The process to contribute a new extractor is like this:**
-
-> [!IMPORTANT]
-> This process is getting much easier after v0.8.x, there is a new plugin system under development: https://github.com/ArchiveBox/ArchiveBox/releases/tag/v0.8.4-rc
-
-1. [Open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new?assignees=&labels=changes%3A+behavior%2Cstatus%3A+idea+phase&template=feature_request.md&title=Feature+Request%3A+...) with your propsoed implementation (please link to the pages of any new external dependencies you plan on using)
-2. Ensure any dependencies needed are easily installable via a package managers like `apt`, `brew`, `pip3`, `npm`
-   (Ideally, prefer to use external programs available via `pip3` or `npm`, however we do support using any binary installable via package manager that exposes a CLI/Python API and writes output to stdout or the filesystem.)
-3. Create a new file in [`archivebox/extractors/EXTRACTOR.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors) (copy an existing extractor like [`singlefile.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/singlefile.py) as a template)
-4. Add config settings to enable/disable any new dependencies and the extractor as a whole, e.g. `USE_DEPENDENCYNAME`, `SAVE_EXTRACTORNAME`, `EXTRACTORNAME_SOMEOTHEROPTION` in [`archivebox/config.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config.py)
-5. Add a preview section to [`archivebox/templates/core/snapshot.html`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) to view the output, and a column to [`archivebox/templates/core/index_row.html`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/index_row.html) with an icon for your extractor
-6. Add an integration test for your extractor in [`tests/test_extractors.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/tests/test_extractors.py)
-7. [Submit your PR for review!](https://github.com/ArchiveBox/ArchiveBox/blob/dev/.github/CONTRIBUTING.md) 🎉
-8. Once merged, please document it in these places and anywhere else you see info about other extractors:
-  - https://github.com/ArchiveBox/ArchiveBox#output-formats
-  - https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles
-  - https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies
-
-<br/><br/>
-
-</details>

 #### Build the docs, pip package, and docker image

@@ -1615,7 +1596,8 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 <a href="https://github.com/sponsors/pirate"><img src="https://img.shields.io/badge/Github_Sponsors-%23B7CDFE.svg"/></a> &nbsp;
 <a href="https://www.patreon.com/theSquashSH"><img src="https://img.shields.io/badge/Patreon-%23DD5D76.svg"/></a> &nbsp;
 <a href="https://paypal.me/NicholasSweeting"><img src="https://img.shields.io/badge/Paypal-%23FFD141.svg"/></a> &nbsp;
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a> &nbsp;
+<a href="https://archivebox-shop.fourthwall.com/"><img src="https://img.shields.io/badge/Merch-%23903851.svg"/></a>
 <br/>
 <sup><i>ArchiveBox operates as a US 501(c)(3) nonprofit <a href="https://en.wikipedia.org/wiki/Fiscal_sponsorship">FSP</a> (sponsored by <a href="https://hackclub.com/hcb?ref=donation">HCB</a>), <a href="https://hcb.hackclub.com/donations/start/archivebox">direct donations</a> are tax-deductible.</i></sup>
 <br/><br/>
@@ -3,4 +3,4 @@ ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E
 select = F,E9,W
 max-line-length = 130
 max-complexity = 10
-exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv
+exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv,data,data*
@@ -9,12 +9,23 @@
 # in a universe that seems indifferent to us."
 # --Norber Weiner

-__package__ = 'archivebox'
+__package__ = "archivebox"

 import os
 import sys
-
 from pathlib import Path
+from typing import Protocol, cast
+
+
+class _ReconfigurableStream(Protocol):
+    def reconfigure(self, *, line_buffering: bool) -> object: ...
+
+
+# Force unbuffered output for real-time logs
+if hasattr(sys.stdout, "reconfigure"):
+    cast(_ReconfigurableStream, sys.stdout).reconfigure(line_buffering=True)
+    cast(_ReconfigurableStream, sys.stderr).reconfigure(line_buffering=True)
+os.environ["PYTHONUNBUFFERED"] = "1"

 ASCII_LOGO = """
 █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
@@ -25,39 +36,85 @@ ASCII_LOGO = """
 ╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
 """

-# make sure PACKAGE_DIR is in sys.path so we can import all subfolders
-# without necessarily waiting for django to load them thorugh INSTALLED_APPS
 PACKAGE_DIR = Path(__file__).resolve().parent
-if str(PACKAGE_DIR) not in sys.path:
-    sys.path.append(str(PACKAGE_DIR))
-os.environ['DJANGO_SETTINGS_MODULE'] = 'core.settings'
+
+# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models
+# # Migrations reference models like 'machine.Binary' which need to be importable
+# if str(PACKAGE_DIR) not in sys.path:
+#     sys.path.append(str(PACKAGE_DIR))
+
+os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
+os.environ["TZ"] = "UTC"

 # detect ArchiveBox user's UID/GID based on data dir ownership
-from .config.permissions import drop_privileges                 # noqa
+from .config.permissions import drop_privileges  # noqa
+
 drop_privileges()

-from .misc.checks import check_not_root, check_io_encoding      # noqa
+from .misc.checks import check_not_root, check_not_inside_source_dir, check_io_encoding  # noqa
+
 check_not_root()
+check_not_inside_source_dir()
 check_io_encoding()

-# print('INSTALLING MONKEY PATCHES')
-from .monkey_patches import *                    # noqa
-# print('DONE INSTALLING MONKEY PATCHES')
+from .config.version import VERSION  # noqa


-# print('LOADING VENDORED LIBRARIES')
-from .vendor import load_vendored_libs           # noqa
-load_vendored_libs()
-# print('DONE LOADING VENDORED LIBRARIES')
-
-
-from .config.constants import CONSTANTS                         # noqa
-from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR    # noqa
-from .config.version import VERSION                             # noqa
-
 __version__ = VERSION
-__author__ = 'Nick Sweeting'
-__license__ = 'MIT'
+__author__ = "ArchiveBox"
+__license__ = "MIT"
+
+
+def __getattr__(name: str):
+    if name == "CONSTANTS":
+        from .config.constants import CONSTANTS
+
+        os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
+        return CONSTANTS
+    if name == "DATA_DIR":
+        from .config.paths import DATA_DIR
+
+        return DATA_DIR
+    if name == "VERSION":
+        return VERSION
+    if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
+        from abx_plugins import get_plugins_dir
+
+        builtin_plugins_dir = Path(get_plugins_dir()).resolve()
+        user_plugins_dir = (
+            Path(
+                os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR")
+                or os.environ.get("USER_PLUGINS_DIR")
+                or os.environ.get("DATA_DIR", os.getcwd()),
+            )
+            / "custom_plugins"
+        )
+        plugins = {
+            "builtin": builtin_plugins_dir,
+            "user": user_plugins_dir,
+        }
+        values = {
+            "BUILTIN_PLUGINS_DIR": builtin_plugins_dir,
+            "USER_PLUGINS_DIR": user_plugins_dir,
+            "ALL_PLUGINS": plugins,
+            "LOADED_PLUGINS": plugins,
+        }
+        return values[name]
+    raise AttributeError(name)
+
+
+__all__ = (
+    "ASCII_LOGO",
+    "ASCII_ICON",
+    "PACKAGE_DIR",
+    "DATA_DIR",
+    "CONSTANTS",
+    "VERSION",
+    "BUILTIN_PLUGINS_DIR",
+    "USER_PLUGINS_DIR",
+    "ALL_PLUGINS",
+    "LOADED_PLUGINS",
+)

 ASCII_ICON = """
 ██████████████████████████████████████████████████████████████████████████████████████████████████ 
@@ -1,19 +1,20 @@
 #!/usr/bin/env python3
-"""This is the main entry point for the ArchiveBox CLI."""
-__package__ = 'archivebox'
+"""This is the entrypoint for python -m archivebox ..."""

-import archivebox      # noqa # make sure monkey patches are applied before anything else
+__package__ = "archivebox"
+
+import archivebox  # noqa # make sure monkey patches are applied before anything else
 import sys

 from .cli import main

 ASCII_LOGO_MINI = r"""
-     _             _     _           ____            
+     _             _     _           ____
    / \   _ __ ___| |__ (_)_   _____| __ )  _____  __
   / _ \ | '__/ __| '_ \| \ \ / / _ \  _ \ / _ \ \/ /
-  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  < 
+  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  <
 /_/   \_\_|  \___|_| |_|_| \_/ \___|____/ \___/_/\_\
 """

-if __name__ == '__main__':
+if __name__ == "__main__":
    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1 +0,0 @@
-__package__ = 'abid_utils'
@@ -1,223 +0,0 @@
-__package__ = 'archivebox.abid_utils'
-
-from typing import NamedTuple, Any, Union, Dict
-
-import ulid
-import uuid6
-import hashlib
-from urllib.parse import urlparse
-
-from uuid import UUID
-from typeid import TypeID            # type: ignore[import-untyped]
-from datetime import datetime
-
-from archivebox.misc.util import enforce_types
-
-
-ABID_PREFIX_LEN = 4
-ABID_SUFFIX_LEN = 26
-ABID_LEN = 30
-ABID_TS_LEN = 10
-ABID_URI_LEN = 8
-ABID_SUBTYPE_LEN = 2
-ABID_RAND_LEN = 6
-
-DEFAULT_ABID_PREFIX = 'obj_'
-
-# allows people to keep their uris secret on a per-instance basis by changing the salt.
-# the default means everyone can share the same namespace for URI hashes,
-# meaning anyone who has a URI and wants to check if you have it can guess the ABID
-DEFAULT_ABID_URI_SALT = '687c2fff14e3a7780faa5a40c237b19b5b51b089'
-
-
-class ABID(NamedTuple):
-    """
-    e.g. ABID('obj_01HX9FPYTRE4A5CCD901ZYEBQE')
-    """
-    prefix: str            # e.g. obj_
-    ts: str                # e.g. 01HX9FPYTR
-    uri: str               # e.g. E4A5CCD9
-    subtype: str           # e.g. 01
-    rand: str              # e.g. ZYEBQE
-    
-    # salt: str = DEFAULT_ABID_URI_SALT
-
-    def __getattr__(self, attr: str) -> Any:
-        return getattr(self.ulid, attr)
-
-    def __eq__(self, other: Any) -> bool:
-        try:
-            return self.ulid == other.ulid
-        except AttributeError:
-            return NotImplemented
-
-    def __str__(self) -> str:
-        return self.prefix + self.suffix
-
-    def __len__(self) -> int:
-        return len(self.prefix + self.suffix)
-
-    @classmethod
-    def parse(cls, buffer: Union[str, UUID, ulid.ULID, TypeID, 'ABID'], prefix=DEFAULT_ABID_PREFIX) -> 'ABID':
-        assert buffer, f'Attempted to create ABID from null value {buffer}'
-
-        buffer = str(buffer)
-        if '_' in buffer:
-            prefix, suffix = buffer.split('_')
-        else:
-            prefix, suffix = prefix.strip('_'), buffer
-
-        assert len(prefix) == ABID_PREFIX_LEN - 1   # length without trailing _
-        assert len(suffix) == ABID_SUFFIX_LEN, f'Suffix {suffix} from {buffer} was not {ABID_SUFFIX_LEN} chars long'
-
-        return cls(
-            prefix=abid_part_from_prefix(prefix),
-            ts=suffix[0:10].upper(),
-            uri=suffix[10:18].upper(),
-            subtype=suffix[18:20].upper(),
-            rand=suffix[20:26].upper(),
-        )
-    
-    @property
-    def uri_salt(self) -> str:
-        return DEFAULT_ABID_URI_SALT
-
-    @property
-    def suffix(self):
-        return ''.join((self.ts, self.uri, self.subtype, self.rand))
-    
-    @property
-    def ulid(self) -> ulid.ULID:
-        return ulid.parse(self.suffix)
-
-    @property
-    def uuid(self) -> UUID:
-        return self.ulid.uuid
-
-    @property
-    def uuid6(self) -> uuid6.UUID:
-        return uuid6.UUID(hex=self.uuid.hex)
-
-    @property
-    def typeid(self) -> TypeID:
-        return TypeID.from_uuid(prefix=self.prefix.strip('_'), suffix=self.uuid6)
-
-    @property
-    def datetime(self) -> datetime:
-        return self.ulid.timestamp().datetime
-
-
-
-####################################################
-
-
-@enforce_types
-def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
-    """
-    https://example.com -> 'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25' (example.com)
-    """
-    if isinstance(uri, bytes):
-        uri_str: str = uri.decode()
-    else:
-        uri_str = str(uri)
-
-    # only hash the domain part of URLs
-    if '://' in uri_str:
-        try:
-            domain = urlparse(uri_str).netloc
-            if domain:
-                uri_str = domain
-        except AttributeError:
-            pass
-    
-    # the uri hash is the sha256 of the domain + salt
-    uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
-
-    return hashlib.sha256(uri_bytes).hexdigest().upper()
-
-@enforce_types
-def abid_part_from_prefix(prefix: str) -> str:
-    """
-    'snp_'
-    """
-    # if prefix is None:
-    #     return 'obj_'
-
-    prefix = prefix.strip('_').lower()
-    assert len(prefix) == 3
-    return prefix + '_'
-
-@enforce_types
-def abid_part_from_uri(uri: Any, salt: str=DEFAULT_ABID_URI_SALT) -> str:
-    """
-    'E4A5CCD9'     # takes first 8 characters of sha256(url)
-    """
-    uri = str(uri).strip()
-    assert uri not in ('None', '')
-    return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
-
-@enforce_types
-def abid_part_from_ts(ts: datetime) -> str:
-    """
-    '01HX9FPYTR'   # produces 10 character Timestamp section of ulid based on added date
-    """
-    return str(ulid.from_timestamp(ts))[:ABID_TS_LEN]
-
-@enforce_types
-def ts_from_abid(abid: str) -> datetime:
-    return ulid.parse(abid.split('_', 1)[-1]).timestamp().datetime
-
-@enforce_types
-def abid_part_from_subtype(subtype: str | int) -> str:
-    """
-    Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
-    Also allows us to change the ulid spec later by putting special sigil values here.
-    """
-    subtype = str(subtype)
-    if len(subtype) == ABID_SUBTYPE_LEN:
-        return subtype
-
-    return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()
-
-@enforce_types
-def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
-    """
-    'ZYEBQE'   # takes last 6 characters of randomness from existing legacy uuid db field
-    """
-    if rand is None:
-        # if it's None we generate a new random 6 character hex string
-        return str(ulid.new())[-ABID_RAND_LEN:]
-    elif isinstance(rand, UUID):
-        # if it's a uuid we take the last 6 characters of the ULID represation of it
-        return str(ulid.from_uuid(rand))[-ABID_RAND_LEN:]
-    elif isinstance(rand, int):
-        # if it's a BigAutoInteger field we convert it from an int to a 0-padded string
-        rand_str = str(rand)[-ABID_RAND_LEN:]
-        padding_needed = ABID_RAND_LEN - len(rand_str)
-        rand_str = ('0'*padding_needed) + rand_str
-        return rand_str
-
-    # otherwise treat it as a string, take the last 6 characters of it verbatim
-    return str(rand)[-ABID_RAND_LEN:].upper()
-
-
-@enforce_types
-def abid_hashes_from_values(prefix: str, ts: datetime, uri: Any, subtype: str | int, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> Dict[str, str]:
-    return {
-        'prefix': abid_part_from_prefix(prefix),
-        'ts': abid_part_from_ts(ts),
-        'uri': abid_part_from_uri(uri, salt=salt),
-        'subtype': abid_part_from_subtype(subtype),
-        'rand': abid_part_from_rand(rand),
-        # 'salt': don't add this, salt combined with uri above to form a single hash
-    }
-
-@enforce_types
-def abid_from_values(prefix: str, ts: datetime, uri: str, subtype: str, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> ABID:
-    """
-    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
-    """
-
-    abid = ABID(**abid_hashes_from_values(prefix, ts, uri, subtype, rand, salt=salt))
-    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
-    return abid
@@ -1,173 +0,0 @@
-__package__ = 'archivebox.abid_utils'
-
-from typing import Any
-
-from django.contrib import admin, messages
-from django.core.exceptions import ValidationError
-from django.utils.html import format_html
-from django.utils.safestring import mark_safe
-from django.shortcuts import redirect
-
-from django_object_actions import DjangoObjectActions, action
-
-
-from api.auth import get_or_create_api_token
-
-from archivebox.misc.util import parse_date
-from .abid import ABID
-
-def highlight_diff(display_val: Any, compare_val: Any, invert: bool=False, color_same: str | None=None, color_diff: str | None=None):
-    """highlight each character in red that differs with the char at the same index in compare_val"""
-
-    display_val = str(display_val)
-    compare_val = str(compare_val)
-
-    if len(compare_val) < len(display_val):
-        compare_val += ' ' * (len(display_val) - len(compare_val))
-
-    similar_color, highlighted_color = color_same or 'inherit', color_diff or 'red'
-    if invert:
-        similar_color, highlighted_color = color_same or 'green', color_diff or 'inherit'
-
-    return mark_safe(''.join(
-        format_html('<span style="color: {};">{}</span>', highlighted_color, display_val[i])
-        if display_val[i] != compare_val[i] else
-        format_html('<span style="color: {};">{}</span>', similar_color, display_val[i])
-        for i in range(len(display_val))
-    ))
-
-def get_abid_info(self, obj, request=None):
-    try:
-        #abid_diff = f' != obj.ABID: {highlight_diff(obj.ABID, obj.abid)} ❌' if str(obj.ABID) != str(obj.abid) else ' == .ABID ✅'
-
-        fresh_values = obj.ABID_FRESH_VALUES
-        fresh_hashes = obj.ABID_FRESH_HASHES
-        fresh_diffs = obj.ABID_FRESH_DIFFS
-        fresh_abid = ABID(**fresh_hashes)
-        
-        fresh_abid_diff = f'❌ != &nbsp; .fresh_abid: {highlight_diff(fresh_abid, obj.ABID)}' if str(fresh_abid) != str(obj.ABID) else '✅'
-        fresh_uuid_diff = f'❌ != &nbsp; .fresh_uuid: {highlight_diff(fresh_abid.uuid, obj.ABID.uuid)}' if str(fresh_abid.uuid) != str(obj.ABID.uuid) else '✅'
-
-        id_pk_diff = f'❌ !=  .pk: {highlight_diff(obj.pk, obj.id)}' if str(obj.pk) != str(obj.id) else '✅'
-
-        fresh_ts = parse_date(fresh_values['ts']) or None
-        ts_diff = f'❌ != {highlight_diff( fresh_hashes["ts"], obj.ABID.ts)}' if  fresh_hashes["ts"] != obj.ABID.ts else '✅'
-
-        derived_uri = fresh_hashes['uri']
-        uri_diff = f'❌ != {highlight_diff(derived_uri, obj.ABID.uri)}' if derived_uri != obj.ABID.uri else '✅'
-
-        derived_subtype = fresh_hashes['subtype']
-        subtype_diff = f'❌ != {highlight_diff(derived_subtype, obj.ABID.subtype)}' if derived_subtype != obj.ABID.subtype else '✅'
-
-        derived_rand = fresh_hashes['rand']
-        rand_diff = f'❌ != {highlight_diff(derived_rand, obj.ABID.rand)}' if derived_rand != obj.ABID.rand else '✅'
-
-        return format_html(
-            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
-            '''
-            <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
-            <br/><hr/>
-            <div style="opacity: 0.8">
-            &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
-            &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
-            &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</small><br/>
-            <hr/>
-            &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; {}</code> &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
-            &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; {}</code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> <span style="display:inline-block; vertical-align: -4px; width: 330px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}: <code style="user-select: all">{}</code></span><br/>
-            &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
-            &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code></code>
-            <br/><hr/>
-            <span style="color: #f375a0">{}</span> <code style="color: red"><b>{}</b></code> {}
-            </div>
-            ''',
-            obj.api_url + (f'?api_key={get_or_create_api_token(request.user)}' if request and request.user else ''), obj.api_url, obj.api_docs_url,
-            highlight_diff(obj.id, obj.ABID.uuid, invert=True), mark_safe(id_pk_diff),
-            highlight_diff(obj.ABID.uuid, obj.id, invert=True), mark_safe(fresh_uuid_diff),
-            highlight_diff(obj.abid, fresh_abid), mark_safe(fresh_abid_diff),
-            # str(fresh_abid.uuid), mark_safe(fresh_uuid_diff),
-            # str(fresh_abid), mark_safe(fresh_abid_diff),
-            highlight_diff(obj.ABID.ts,  fresh_hashes['ts']), highlight_diff(str(obj.ABID.uuid)[0:14], str(fresh_abid.uuid)[0:14]), mark_safe(ts_diff), obj.abid_ts_src, fresh_ts and fresh_ts.isoformat(),
-            highlight_diff(obj.ABID.uri, derived_uri), highlight_diff(str(obj.ABID.uuid)[14:26], str(fresh_abid.uuid)[14:26]), mark_safe(uri_diff), obj.abid_uri_src, str(fresh_values['uri']),
-            highlight_diff(obj.ABID.subtype, derived_subtype), highlight_diff(str(obj.ABID.uuid)[26:28], str(fresh_abid.uuid)[26:28]), mark_safe(subtype_diff), obj.abid_subtype_src, str(fresh_values['subtype']),
-            highlight_diff(obj.ABID.rand, derived_rand), highlight_diff(str(obj.ABID.uuid)[28:36], str(fresh_abid.uuid)[28:36]), mark_safe(rand_diff), obj.abid_rand_src, str(fresh_values['rand'])[-7:],
-            'Some values the ABID depends on have changed since the ABID was issued:' if fresh_diffs else '',
-            ", ".join(diff['abid_src'] for diff in fresh_diffs.values()),
-            '(clicking "Regenerate ABID" in the upper right will assign a new ABID, breaking any external references to the old ABID)' if fresh_diffs else '',
-        )
-    except Exception as e:
-        # import ipdb; ipdb.set_trace()
-        return str(e)
-
-
-class ABIDModelAdmin(DjangoObjectActions, admin.ModelAdmin):
-    list_display = ('created_at', 'created_by', 'abid')
-    sort_fields = ('created_at', 'created_by', 'abid')
-    readonly_fields = ('created_at', 'modified_at', 'abid_info')
-    # fields = [*readonly_fields]
-    
-    change_actions = ("regenerate_abid",)
-    # changelist_actions = ("regenerate_abid",)
-
-    def _get_obj_does_not_exist_redirect(self, request, opts, object_id):
-        try:
-            object_pk = self.model.id_from_abid(object_id)
-            return redirect(self.request.path.replace(object_id, object_pk), permanent=False)
-        except (self.model.DoesNotExist, ValidationError):
-            pass
-        return super()._get_obj_does_not_exist_redirect(request, opts, object_id)       # type: ignore
-    
-    def queryset(self, request):
-        self.request = request
-        return super().queryset(request)                                                # type: ignore
-    
-    def change_view(self, request, object_id, form_url="", extra_context=None):
-        self.request = request
-        return super().change_view(request, object_id, form_url, extra_context)
-
-    def get_form(self, request, obj=None, **kwargs):
-        self.request = request
-        form = super().get_form(request, obj, **kwargs)
-        if 'created_by' in form.base_fields:
-            form.base_fields['created_by'].initial = request.user
-            
-        if obj:
-            if obj.ABID_FRESH_DIFFS:
-                messages.warning(request, "The ABID is not in sync with the object! See the API Identifiers section below for more info...")
-
-        return form
-
-    def get_formset(self, request, formset=None, obj=None, **kwargs):
-        formset = super().get_formset(request, formset, obj, **kwargs)                  # type: ignore
-        formset.form.base_fields['created_at'].disabled = True
-        
-        return formset
-
-    def save_model(self, request, obj, form, change):
-        self.request = request
-
-        old_abid = getattr(obj, '_previous_abid', None) or obj.abid
-
-        super().save_model(request, obj, form, change)
-        obj.refresh_from_db()
-
-        new_abid = obj.abid
-        if new_abid != old_abid:
-            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
-        # import ipdb; ipdb.set_trace()
-
-    @admin.display(description='API Identifiers')
-    def abid_info(self, obj):
-        return get_abid_info(self, obj, request=self.request)
-
-    @action(label="Regenerate ABID", description="Re-Generate the ABID based on fresh values")
-    def regenerate_abid(self, request, obj):
-        old_abid = str(obj.abid)
-        obj.abid = obj.issue_new_abid(overwrite=True)
-        obj.save()
-        obj.refresh_from_db()
-        new_abid = str(obj.abid)
-
-        if new_abid != old_abid:
-            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
-        else:
-            messages.success(request, "The ABID was not regenerated, it is already up-to-date with the object.")
@@ -1,7 +0,0 @@
-from django.apps import AppConfig
-
-
-class AbidUtilsConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    
-    name = 'abid_utils'
@@ -1,470 +0,0 @@
-"""
-This file provides the Django ABIDField and ABIDModel base model to inherit from.
-"""
-
-
-from typing import Any, Dict, Union, List, Set, cast
-
-from uuid import uuid4
-from functools import partial
-from charidfield import CharIDField  # type: ignore[import-untyped]
-
-from django.contrib import admin
-from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
-from django.db import models
-from django.utils import timezone
-from django.db.utils import OperationalError
-from django.contrib.auth import get_user_model
-from django.urls import reverse_lazy
-
-from django_stubs_ext.db.models import TypedModelMeta
-
-from .abid import (
-    ABID,
-    ABID_LEN,
-    ABID_RAND_LEN,
-    ABID_SUFFIX_LEN,
-    DEFAULT_ABID_PREFIX,
-    DEFAULT_ABID_URI_SALT,
-    abid_part_from_prefix,
-    abid_hashes_from_values,
-    ts_from_abid,
-    abid_part_from_ts,
-)
-
-####################################################
-
-
-# Database Field for typeid/ulid style IDs with a prefix, e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ
-ABIDField = partial(
-    CharIDField,
-    max_length=ABID_LEN,
-    help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
-    default=None,
-    null=True,
-    blank=True,
-    db_index=True,
-    unique=True,
-)
-
-def get_or_create_system_user_pk(username='system'):
-    """Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
-
-    User = get_user_model()
-
-    # if only one user exists total, return that user
-    if User.objects.filter(is_superuser=True).count() == 1:
-        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
-
-    # otherwise, create a dedicated "system" user
-    user, _was_created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
-    return user.pk
-
-
-class AutoDateTimeField(models.DateTimeField):
-    # def pre_save(self, model_instance, add):
-    #     return timezone.now()
-    pass
-
-class ABIDError(Exception):
-    pass
-
-
-class ABIDModel(models.Model):
-    """
-    Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
-    """
-    abid_prefix: str = DEFAULT_ABID_PREFIX            # e.g. 'tag_'
-    abid_ts_src = 'self.created_at'                  # e.g. 'self.created_at'
-    abid_uri_src = 'None'                            # e.g. 'self.uri'                (MUST BE SET)
-    abid_subtype_src = 'self.__class__.__name__'     # e.g. 'self.extractor'
-    abid_rand_src = 'self.id'                        # e.g. 'self.uuid' or 'self.id'
-    abid_salt: str = DEFAULT_ABID_URI_SALT           # combined with self.uri to anonymize hashes on a per-install basis (default is shared globally with all users, means everyone will hash ABC to -> 123 the same around the world, makes it easy to share ABIDs across installs and see if they are for the same URI. Change this if you dont want your hashes to be guessable / in the same hash space as all other users)
-    abid_drift_allowed: bool = False                 # set to True to allow abid_field values to change after a fixed ABID has been issued (NOT RECOMMENDED: means values can drift out of sync from original ABID)
-
-    # id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
-    # abid = ABIDField(prefix=abid_prefix)
-
-    # created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    # created_at = AutoDateTimeField(default=None, null=False, db_index=True)
-    # modified_at = models.DateTimeField(auto_now=True)
-
-    _prefetched_objects_cache: Dict[str, Any]
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    @admin.display(description='Summary')
-    def __str__(self) -> str:
-        return f'[{self.abid or (self.abid_prefix + "NEW")}] {self.__class__.__name__} {eval(self.abid_uri_src)}'
-
-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        """Overriden __init__ method ensures we have a stable creation timestamp that fields can use within initialization code pre-saving to DB."""
-        super().__init__(*args, **kwargs)
-        # pre-compute a stable timestamp of the obj init time (with abid.ts precision limit applied) for use when object is first created,
-        # some other fields depend on a timestamp at creation time, and it's nice to have one common timestamp they can all share.
-        # Used as an alternative to auto_now_add=True + auto_now=True which can produce two different times & requires saving to DB to get the TS.
-        # (ordinarily fields cant depend on other fields until the obj is saved to db and recalled)
-        self._init_timestamp = ts_from_abid(abid_part_from_ts(timezone.now()))
-
-    def clean(self, abid_drift_allowed: bool | None=None) -> None:
-        if self._state.adding:
-            # only runs once when a new object is first saved to the DB
-            # sets self.id, self.pk, self.created_by, self.created_at, self.modified_at
-            self._previous_abid = None
-            self.abid = str(self.issue_new_abid())
-
-        else:
-            # otherwise if updating, make sure none of the field changes would invalidate existing ABID
-            abid_diffs = self.ABID_FRESH_DIFFS
-            if abid_diffs:
-                # change has invalidated the existing ABID, raise a nice ValidationError pointing out which fields caused the issue
-
-                keys_changed = ', '.join(diff['abid_src'] for diff in abid_diffs.values())
-                full_summary = (
-                    f"This {self.__class__.__name__}(abid={str(self.ABID)}) was assigned a fixed, unique ID (ABID) based on its contents when it was created. " +
-                    f"\nYou must reduce your changes to not affect these fields [{keys_changed}], or create a new {self.__class__.__name__} object instead."
-                )
-
-                change_error = ValidationError({
-                    **{
-                        # url: ValidationError('Cannot update self.url= https://example.com/old -> https://example.com/new ...')
-                        diff['abid_src'].replace('self.', '')
-                            if (diff['old_val'] != diff['new_val']) and hasattr(self, diff['abid_src'].replace('self.', ''))
-                            else NON_FIELD_ERRORS
-                        : ValidationError(
-                            'Cannot update %(abid_src)s= "%(old_val)s" -> "%(new_val)s" (would alter %(model)s.ABID.%(key)s=%(old_hash)s to %(new_hash)s)',
-                            code='ABIDConflict',
-                            params=diff,
-                        )
-                        for diff in abid_diffs.values()
-                    },
-                    NON_FIELD_ERRORS: ValidationError(full_summary),
-                })
-
-                allowed_to_invalidate_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed
-                if allowed_to_invalidate_abid:
-                    print(f'\n#### WARNING: Change allowed despite it invalidating the ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})!', self.abid)
-                    print(change_error)
-                    print('--------------------------------------------------------------------------------------------------')
-                else:
-                    print(f'\n#### ERROR:   Change blocked because it would invalidate ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})', self.abid)
-                    print(change_error)
-                    print('--------------------------------------------------------------------------------------------------')
-                    raise change_error
-
-    def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None:
-        """Overriden save method ensures new ABID is generated while a new object is first saving."""
-
-        self.clean(abid_drift_allowed=abid_drift_allowed)
-
-        return super().save(*args, **kwargs)
-    
-    @classmethod
-    def id_from_abid(cls, abid: str) -> str:
-        return str(cls.objects.only('pk').get(abid=cls.abid_prefix + str(abid).split('_', 1)[-1]).pk)
-
-    @property
-    def ABID_SOURCES(self) -> Dict[str, str]:
-        """"Get the dict of fresh ABID component values based on the live object's properties."""
-        assert self.abid_prefix
-        return {
-            'prefix': 'self.abid_prefix',             # defined as static class vars at build time
-            'ts': self.abid_ts_src,
-            'uri': self.abid_uri_src,
-            'subtype': self.abid_subtype_src,
-            'rand': self.abid_rand_src,
-            'salt': 'self.abid_salt',               # defined as static class vars at build time
-        }
-
-    @property
-    def ABID_FRESH_VALUES(self) -> Dict[str, Any]:
-        """"Get the dict of fresh ABID component values based on the live object's properties."""
-        abid_sources = self.ABID_SOURCES
-        assert all(src != 'None' for src in abid_sources.values())
-        return {
-            'prefix': eval(abid_sources['prefix']),
-            'ts': eval(abid_sources['ts']),
-            'uri': eval(abid_sources['uri']),
-            'subtype': eval(abid_sources['subtype']),
-            'rand': eval(abid_sources['rand']),
-            'salt': eval(abid_sources['salt']),
-        }
-    
-    @property
-    def ABID_FRESH_HASHES(self) -> Dict[str, str]:
-        """"Get the dict of fresh ABID component hashes based on the live object's properties."""
-        abid_values = self.ABID_FRESH_VALUES
-        assert all(val for val in abid_values.values())
-        return abid_hashes_from_values(
-            prefix=abid_values['prefix'],
-            ts=abid_values['ts'],
-            uri=abid_values['uri'],
-            subtype=abid_values['subtype'],
-            rand=abid_values['rand'],
-            salt=abid_values['salt'],
-        )
-    
-    @property
-    def ABID_FRESH_DIFFS(self) -> Dict[str, Dict[str, Any]]:
-        """Get the dict of discrepancies between the existing saved ABID and a new fresh ABID computed based on the live object."""
-        existing_abid = self.ABID
-        existing_values = {} if self._state.adding else self.__class__.objects.get(pk=self.pk).ABID_FRESH_VALUES
-        abid_sources = self.ABID_SOURCES
-        fresh_values = self.ABID_FRESH_VALUES
-        fresh_hashes = self.ABID_FRESH_HASHES
-        return {
-            key: {
-                'key': key,
-                'model': self.__class__.__name__,
-                'pk': self.pk,
-                'abid_src': abid_sources[key],
-                'old_val': existing_values.get(key, None),
-                'old_hash': getattr(existing_abid, key),
-                'new_val': fresh_values[key],
-                'new_hash': new_hash,
-                'summary': f'{abid_sources[key]}= "{existing_values.get(key, None)}" -> "{fresh_values[key]}" (would alter {self.__class__.__name__.lower()}.ABID.{key}={getattr(existing_abid, key)} to {new_hash})',
-            }
-            for key, new_hash in fresh_hashes.items()
-            if getattr(existing_abid, key) != new_hash
-        }
-
-    def issue_new_abid(self, overwrite=False) -> ABID:
-        """
-        Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB).
-        """
-        if not overwrite:
-            assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
-        assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'
-
-        # Setup Field defaults to be ready for ABID generation
-        self.abid = None
-        self.id = self.id or uuid4()
-        self.pk = self.id
-        self.created_at = self.created_at or self._init_timestamp  # cut off precision to match precision of TS component
-        self.modified_at = self.modified_at or self.created_at
-        self.created_by_id = (hasattr(self, 'created_by_id') and self.created_by_id) or get_or_create_system_user_pk()
-        
-        # Compute fresh ABID values & hashes based on object's live properties
-        abid_fresh_values = self.ABID_FRESH_VALUES
-        assert all(abid_fresh_values.values()), f'All ABID_FRESH_VALUES must be set {abid_fresh_values}'
-        abid_fresh_hashes = self.ABID_FRESH_HASHES
-        assert all(abid_fresh_hashes.values()), f'All ABID_FRESH_HASHES must be able to be generated {abid_fresh_hashes}'
-        
-        new_abid = ABID(**abid_fresh_hashes)
-        
-        assert new_abid.ulid and new_abid.uuid and new_abid.typeid, f'Failed to calculate {abid_fresh_values["prefix"]}_ABID for {self.__class__.__name__}'
-
-        return new_abid
-
-    @property
-    def ABID(self) -> ABID:
-        """
-        Get the object's existing ABID (from self.abid if it's already saved to DB, otherwise generated fresh)
-        e.g. -> ABID(ts='01HX9FPYTR', uri='E4A5CCD9', subtype='00', rand='ZYEBQE')
-        """
-
-        if self.abid:
-            return ABID.parse(cast(str, self.abid))
-        
-        return self.issue_new_abid()
-
-    # These are all example helpers to make it easy to access alternate formats of the ABID.*, only add them if you actually need them
-    # @property
-    # def UUID(self) -> UUID:
-    #     """
-    #     Get a uuid.UUID (v4) representation of the object's ABID.
-    #     """
-    #     return self.ABID.uuid
-    
-    # @property
-    # def uuid(self) -> str:
-    #     """
-    #     Get a str uuid.UUID (v4) representation of the object's ABID.
-    #     """
-    #     return str(self.ABID.uuid)
-    
-    # @property
-    # def ULID(self) -> ULID:
-    #     """
-    #     Get a ulid.ULID representation of the object's ABID.
-    #     """
-    #     return self.ABID.ulid
-
-    # @property
-    # def TypeID(self) -> TypeID:
-    #     """
-    #     Get a typeid.TypeID (stripe-style) representation of the object's ABID.
-    #     """
-    #     return self.ABID.typeid
-    
-    @property
-    def api_url(self) -> str:
-        """
-        Compute the REST API URL to access this object.
-        e.g. /api/v1/core/snapshot/snp_01BJQMF54D093DXEAWZ6JYRP
-        """
-        return reverse_lazy('api-1:get_any', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
-
-    @property
-    def api_docs_url(self) -> str:
-        """
-        Compute the REST API Documentation URL to learn about accessing this object.
-        e.g. /api/v1/docs#/Core%20Models/api_v1_core_get_snapshots
-        """
-        return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}'
-
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
-    def get_absolute_url(self):
-        return self.api_docs_url
-
-####################################################
-
-# Django helpers
-def find_all_abid_prefixes() -> Dict[str, type[models.Model]]:
-    """
-    Return the mapping of all ABID prefixes to their models.
-    e.g. {'tag_': core.models.Tag, 'snp_': core.models.Snapshot, ...}
-    """
-    import django.apps
-    prefix_map = {}
-
-    for model in django.apps.apps.get_models():
-        abid_prefix = getattr(model, 'abid_prefix', None)
-        if abid_prefix:
-            prefix_map[abid_prefix] = model
-    return prefix_map
-
-def find_prefix_for_abid(abid: ABID) -> str:
-    """
-    Find the correct prefix for a given ABID that may have be missing a prefix (slow).
-    e.g. ABID('obj_01BJQMF54D093DXEAWZ6JYRPAQ') -> 'snp_'
-    """
-    # if existing abid prefix is correct, lookup is easy
-    model = find_model_from_abid(abid)
-    if model:
-        assert issubclass(model, ABIDModel)
-        return model.abid_prefix
-
-    # prefix might be obj_ or missing, fuzzy-search to find any object that matches
-    return find_obj_from_abid_rand(abid)[0].abid_prefix
-
-def find_model_from_abid_prefix(prefix: str) -> type[ABIDModel] | None:
-    """
-    Return the Django Model that corresponds to a given ABID prefix.
-    e.g. 'tag_' -> core.models.Tag
-    """
-    prefix = abid_part_from_prefix(prefix)
-
-    import django.apps
-
-    for model in django.apps.apps.get_models():
-        if not issubclass(model, ABIDModel): continue   # skip non-ABID-enabled models
-        if not hasattr(model, 'objects'): continue      # skip abstract models
-
-        if (model.abid_prefix == prefix):
-            return model
-
-    return None
-
-def find_model_from_abid(abid: ABID) -> type[models.Model] | None:
-    """
-    Shortcut for find_model_from_abid_prefix(abid.prefix)
-    """
-    return find_model_from_abid_prefix(abid.prefix)
-
-def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDModel]:
-    """
-    Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
-    e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
-    Honestly should only be used for debugging, no reason to expose this ability to users.
-    """
-
-    # convert str to ABID if necessary
-    if isinstance(rand, ABID):
-        abid: ABID = rand
-    else:
-        rand = str(rand)
-        if len(rand) < ABID_SUFFIX_LEN:
-            padding_needed = ABID_SUFFIX_LEN - len(rand)
-            rand = ('0'*padding_needed) + rand
-        abid = ABID.parse(rand)
-
-    import django.apps
-
-    partial_matches: List[ABIDModel] = []
-
-    models_to_try = cast(Set[type[models.Model]], set(filter(bool, (
-        model,
-        find_model_from_abid(abid),
-        *django.apps.apps.get_models(),
-    ))))
-    # print(abid, abid.rand, abid.uuid, models_to_try)
-
-    for model in models_to_try:
-        if not issubclass(model, ABIDModel): continue   # skip Models that arent ABID-enabled
-        if not hasattr(model, 'objects'): continue      # skip abstract Models
-        assert hasattr(model, 'objects')                # force-fix for type hint nit about missing manager https://github.com/typeddjango/django-stubs/issues/1684
-
-        # continue on to try fuzzy searching by randomness portion derived from uuid field
-        try:
-            qs = []
-            if hasattr(model, 'abid'):
-                qs = model.objects.filter(abid__endswith=abid.rand)
-            elif hasattr(model, 'uuid'):
-                qs = model.objects.filter(uuid__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
-            elif hasattr(model, 'id'):
-                # NOTE: this only works on SQLite where every column is a string
-                # other DB backends like postgres dont let you do __endswith if this is a BigAutoInteger field
-                
-                # try to search for uuid=...-2354352
-                # try to search for id=...2354352
-                # try to search for id=2354352
-                qs = model.objects.filter(
-                    models.Q(id__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
-                    | models.Q(id__endswith=abid.rand)
-                    | models.Q(id__startswith=str(int(abid.rand)) if abid.rand.isdigit() else abid.rand)
-                )
-
-            for obj in qs:
-                if abid in (str(obj.ABID), str(obj.id), str(obj.pk), str(obj.abid)):
-                    # found exact match, no need to keep iterating
-                    return [obj]
-                partial_matches.append(obj)
-        except OperationalError as err:
-            print(f'[!] WARNING: Got error while trying to iterate through QuerySet for {model}:', err, '\n')
-
-    return partial_matches
-
-def find_obj_from_abid(abid: ABID, model=None, fuzzy=False) -> Any:
-    """
-    Find an object with a given ABID by filtering possible models for a matching abid/uuid/id (fast).
-    e.g. 'snp_01BJQMF54D093DXEAWZ6JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
-    """
-
-    model = model or find_model_from_abid(abid)
-    assert model, f'Could not find model that could match this ABID type: {abid}'
-
-    try:
-        if hasattr(model, 'abid'):
-            return model.objects.get(abid__endswith=abid.suffix)
-        if hasattr(model, 'uuid'):
-            return model.objects.get(uuid=abid.uuid)
-        return model.objects.get(id=abid.uuid)
-    except model.DoesNotExist:
-        # if the model has an abid field then it shouldve matched, pointless to fuzzy search in that case
-        if hasattr(model, 'abid') or (not fuzzy):
-            raise
-
-    # continue on to try fuzzy searching by randomness portion derived from uuid field
-    match_by_rand = find_obj_from_abid_rand(abid, model=model)
-    if match_by_rand:
-        if match_by_rand[0].abid_prefix != abid.prefix:
-            print(f'[!] WARNING: fetched object {match_by_rand} even though prefix {abid.prefix} doesnt match!', abid, '\n')
-        return match_by_rand
-
-    raise model.DoesNotExist
-
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
@@ -1,90 +0,0 @@
-__package__ = 'abx'
-
-import importlib
-from pathlib import Path
-from typing import Dict
-
-from . import hookspec as base_spec
-from .hookspec import hookimpl, hookspec           # noqa
-from .manager import pm, PluginManager             # noqa
-
-
-pm.add_hookspecs(base_spec)
-
-
-###### PLUGIN DISCOVERY AND LOADING ########################################################
-
-def get_plugin_order(plugin_entrypoint: Path):
-    order = 999
-    try:
-        # if .plugin_order file exists, use it to set the load priority
-        order = int((plugin_entrypoint.parent / '.plugin_order').read_text())
-    except FileNotFoundError:
-        pass
-    return (order, plugin_entrypoint)
-
-def register_hookspecs(hookspecs):
-    for hookspec_import_path in hookspecs:
-        hookspec_module = importlib.import_module(hookspec_import_path)
-        pm.add_hookspecs(hookspec_module)
-
-
-def find_plugins_in_dir(plugins_dir: Path, prefix: str) -> Dict[str, Path]:
-    return {
-        f"{prefix}.{plugin_entrypoint.parent.name}": plugin_entrypoint.parent
-        for plugin_entrypoint in sorted(plugins_dir.glob("*/apps.py"), key=get_plugin_order)
-    }   # "plugins_pkg.pip": "/app/archivebox/plugins_pkg/pip"
-
-
-def get_pip_installed_plugins(group='abx'):
-    """replaces pm.load_setuptools_entrypoints("abx")"""
-    import importlib.metadata
-
-    DETECTED_PLUGINS = {}   # module_name: module_dir_path
-    for dist in list(importlib.metadata.distributions()):
-        for entrypoint in dist.entry_points:
-            if entrypoint.group != group or pm.is_blocked(entrypoint.name):
-                continue
-            DETECTED_PLUGINS[entrypoint.name] = Path(entrypoint.load().__file__).parent
-            # pm.register(plugin, name=ep.name)
-            # pm._plugin_distinfo.append((plugin, DistFacade(dist)))
-    return DETECTED_PLUGINS
-
-
-def get_plugins_in_dirs(plugin_dirs: Dict[str, Path]):
-    DETECTED_PLUGINS = {}
-    for plugin_prefix, plugin_dir in plugin_dirs.items():
-        DETECTED_PLUGINS.update(find_plugins_in_dir(plugin_dir, prefix=plugin_prefix))
-    return DETECTED_PLUGINS
-
-
-# Load all plugins from pip packages, archivebox built-ins, and user plugins
-
-def load_plugins(plugins_dict: Dict[str, Path]):
-    LOADED_PLUGINS = {}
-    for plugin_module, plugin_dir in plugins_dict.items():
-        # print(f'Loading plugin: {plugin_module} from {plugin_dir}')
-        plugin_module_loaded = importlib.import_module(plugin_module)
-        pm.register(plugin_module_loaded)
-        LOADED_PLUGINS[plugin_module] = plugin_module_loaded.PLUGIN
-        # print(f'    √ Loaded plugin: {plugin_module}')
-    return LOADED_PLUGINS
-
-def get_registered_plugins():
-    plugins = {}
-    plugin_to_distinfo = dict(pm.list_plugin_distinfo())
-    for plugin in pm.get_plugins():
-        plugin_info = {
-            "name": plugin.__name__,
-            "hooks": [h.name for h in pm.get_hookcallers(plugin) or ()],
-        }
-        distinfo = plugin_to_distinfo.get(plugin)
-        if distinfo:
-            plugin_info["version"] = distinfo.version
-            plugin_info["name"] = (
-                getattr(distinfo, "name", None) or distinfo.project_name
-            )
-        plugins[plugin_info["name"]] = plugin_info
-    return plugins
-
-
@@ -1,40 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import os
-import importlib
-
-from typing import Dict
-from pathlib import Path
-
-
-def load_archivebox_plugins(pm, plugins_dict: Dict[str, Path]):
-    """Load archivebox plugins, very similar to abx.load_plugins but it looks for a pydantic PLUGIN model + hooks in apps.py"""
-    LOADED_PLUGINS = {}
-    for plugin_module, plugin_dir in plugins_dict.items():
-        # print(f'Loading plugin: {plugin_module} from {plugin_dir}')
-        
-        archivebox_plugins_found = []
-        
-        # 1. register the plugin module directly in case it contains any look hookimpls (e.g. in __init__.py)
-        plugin_module_loaded = importlib.import_module(plugin_module)
-        pm.register(plugin_module_loaded)
-        if hasattr(plugin_module_loaded, 'PLUGIN'):
-            archivebox_plugins_found.append(plugin_module_loaded.PLUGIN)
-        
-        # 2. then try to import plugin_module.apps as well
-        if os.access(plugin_dir / 'apps.py', os.R_OK):
-            plugin_apps = importlib.import_module(plugin_module + '.apps')
-            pm.register(plugin_apps)                                           # register the whole .apps  in case it contains loose hookimpls (not in a class)
-            if hasattr(plugin_apps, 'PLUGIN'):
-                archivebox_plugins_found.append(plugin_apps.PLUGIN)
-        
-        # 3. then try to look for plugin_module.PLUGIN and register it + all its hooks
-        for ab_plugin in archivebox_plugins_found:
-            pm.register(ab_plugin)
-            for hook in ab_plugin.hooks:
-                hook.__signature__ = hook.__class__.__signature__              # fix to make pydantic model usable as Pluggy plugin
-                pm.register(hook)
-            LOADED_PLUGINS[plugin_module] = ab_plugin
-            
-        # print(f'    √ Loaded plugin: {LOADED_PLUGINS}')
-    return LOADED_PLUGINS
@@ -1,38 +0,0 @@
-__package__ = 'abx.archivebox'
-
-from typing import Dict
-
-import abx
-
-from .base_hook import BaseHook, HookType
-
-
-class BaseAdminDataView(BaseHook):
-    hook_type: HookType = "ADMINDATAVIEW"
-    
-    name: str = 'example_admin_data_view_list'
-    verbose_name: str = 'Data View'
-    route: str = '/__OVERRIDE_THIS__/'
-    view: str = 'plugins_example.example.views.example_view_list'
-    
-    items: Dict[str, str] = {
-        'route': '<str:key>/',
-        "name": 'example_admin_data_view_item',
-        'view': 'plugins_example.example.views.example_view_item',
-    }
-    
-    @abx.hookimpl
-    def get_ADMINDATAVIEWS(self):
-        return [self]
-    
-    @abx.hookimpl
-    def get_ADMIN_DATA_VIEWS_URLS(self):
-        """routes to be added to django.conf.settings.ADMIN_DATA_VIEWS['urls']"""
-        route = {
-            "route": self.route,
-            "view": self.view,
-            "name": self.verbose_name,
-            "items": self.items,
-        }
-        return [route]
-
@@ -1,133 +0,0 @@
-__package__ = "abx.archivebox"
-
-import os
-from typing import Dict, List, Optional
-from typing_extensions import Self
-
-from pydantic import Field, InstanceOf, validate_call
-from pydantic_pkgr import (
-    Binary,
-    BinProvider,
-    BinProviderName,
-    ProviderLookupDict,
-    AptProvider,
-    BrewProvider,
-    EnvProvider,
-)
-
-from archivebox.config import CONSTANTS
-from archivebox.config.permissions import ARCHIVEBOX_USER
-
-import abx
-from .base_hook import BaseHook, HookType
-
-
-class BaseBinProvider(BaseHook, BinProvider):
-    hook_type: HookType = "BINPROVIDER"
-
-    # def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
-    #     Class = super()
-    #     get_abspath_func = lambda: Class.on_get_abspath(bin_name, **context)
-    #     # return cache.get_or_set(f'bin:abspath:{bin_name}', get_abspath_func)
-    #     return get_abspath_func()
-
-    # def on_get_version(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, **context) -> SemVer | None:
-    #     Class = super()
-    #     get_version_func = lambda: Class.on_get_version(bin_name, abspath, **context)
-    #     # return cache.get_or_set(f'bin:version:{bin_name}:{abspath}', get_version_func)
-    #     return get_version_func()
-
-    
-    # TODO: add install/load/load_or_install methods as abx.hookimpl methods
-    
-    @property
-    def admin_url(self) -> str:
-        # e.g. /admin/environment/binproviders/NpmBinProvider/   TODO
-        return "/admin/environment/binaries/"
-
-    @abx.hookimpl
-    def get_BINPROVIDERS(self):
-        return [self]
-
-class BaseBinary(BaseHook, Binary):
-    hook_type: HookType = "BINARY"
-
-    binproviders_supported: List[InstanceOf[BinProvider]] = Field(default_factory=list, alias="binproviders")
-    provider_overrides: Dict[BinProviderName, ProviderLookupDict] = Field(default_factory=dict, alias="overrides")
-
-    @staticmethod
-    def symlink_to_lib(binary, bin_dir=None) -> None:
-        bin_dir = bin_dir or CONSTANTS.LIB_BIN_DIR
-        
-        if not (binary.abspath and os.access(binary.abspath, os.R_OK)):
-            return
-        
-        try:
-            bin_dir.mkdir(parents=True, exist_ok=True)
-            symlink = bin_dir / binary.name
-            symlink.unlink(missing_ok=True)
-            symlink.symlink_to(binary.abspath)
-            symlink.chmod(0o777)   # make sure its executable by everyone
-        except Exception as err:
-            # print(f'[red]:warning: Failed to symlink {symlink} -> {binary.abspath}[/red] {err}')
-            # not actually needed, we can just run without it
-            pass
-        
-    @validate_call
-    def load(self, fresh=False, **kwargs) -> Self:
-        if fresh:
-            binary = super().load(**kwargs)
-            self.symlink_to_lib(binary=binary, bin_dir=CONSTANTS.LIB_BIN_DIR)
-        else:
-            # get cached binary from db
-            try:
-                from machine.models import InstalledBinary
-                installed_binary = InstalledBinary.objects.get_from_db_or_cache(self)
-                binary = InstalledBinary.load_from_db(installed_binary)
-            except Exception:
-                # maybe we are not in a DATA dir so there is no db, fallback to reading from fs
-                # (e.g. when archivebox version is run outside of a DATA dir)
-                binary = super().load(**kwargs)
-        return binary
-    
-    @validate_call
-    def install(self, **kwargs) -> Self:
-        binary = super().install(**kwargs)
-        self.symlink_to_lib(binary=binary, bin_dir=CONSTANTS.LIB_BIN_DIR)
-        return binary
-    
-    @validate_call
-    def load_or_install(self, fresh=False, **kwargs) -> Self:
-        try:
-            binary = self.load(fresh=fresh)
-            if binary and binary.version:
-                self.symlink_to_lib(binary=binary, bin_dir=CONSTANTS.LIB_BIN_DIR)
-                return binary
-        except Exception:
-            pass
-        return self.install(**kwargs)
-    
-    @property
-    def admin_url(self) -> str:
-        # e.g. /admin/environment/config/LdapConfig/
-        return f"/admin/environment/binaries/{self.name}/"
-
-    @abx.hookimpl
-    def get_BINARIES(self):
-        return [self]
-
-
-class AptBinProvider(AptProvider, BaseBinProvider):
-    name: BinProviderName = "apt"
-    
-class BrewBinProvider(BrewProvider, BaseBinProvider):
-    name: BinProviderName = "brew"
-    
-class EnvBinProvider(EnvProvider, BaseBinProvider):
-    name: BinProviderName = "env"
-    
-    euid: Optional[int] = ARCHIVEBOX_USER
-
-apt = AptBinProvider()
-brew = BrewBinProvider()
-env = EnvBinProvider()
@@ -1,44 +0,0 @@
-__package__ = "abx.archivebox"
-
-from typing import List
-
-from django.core.checks import Warning, Tags, register
-
-import abx
-
-from .base_hook import BaseHook, HookType
-
-
-class BaseCheck(BaseHook):
-    hook_type: HookType = "CHECK"
-    
-    tag: str = Tags.database
-    
-    @staticmethod
-    def check(settings, logger) -> List[Warning]:
-        """Override this method to implement your custom runtime check."""
-        errors = []
-        # if not hasattr(settings, 'SOME_KEY'):
-        #     errors.extend(Error(
-        #         'Missing settings.SOME_KEY after django_setup(), did SOME_KEY get loaded?',
-        #         id='core.C001',
-        #         hint='Make sure to run django_setup() is able to load settings.SOME_KEY.',
-        #     ))
-        # logger.debug('[√] Loaded settings.PLUGINS succesfully.')
-        return errors
-
-    @abx.hookimpl
-    def get_CHECKS(self):
-        return [self]
-
-    @abx.hookimpl
-    def register_checks(self):
-        """Tell django that this check exists so it can be run automatically by django."""
-        def run_check(**kwargs):
-            from django.conf import settings
-            import logging
-            return self.check(settings, logging.getLogger("checks"))
-        
-        run_check.__name__ = self.id
-        run_check.tags = [self.tag]
-        register(self.tag)(run_check)
@@ -1,229 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import os
-from pathlib import Path
-from typing import Type, Tuple, Callable, ClassVar
-
-from benedict import benedict
-from pydantic import model_validator, TypeAdapter
-from pydantic_settings import BaseSettings, SettingsConfigDict, PydanticBaseSettingsSource
-from pydantic_settings.sources import TomlConfigSettingsSource
-
-from pydantic_pkgr.base_types import func_takes_args_or_kwargs
-
-import abx
-
-from .base_hook import BaseHook, HookType
-from . import toml_util
-
-
-PACKAGE_DIR = Path(__file__).resolve().parent.parent
-DATA_DIR = Path(os.getcwd()).resolve()
-
-
-
-class FlatTomlConfigSettingsSource(TomlConfigSettingsSource):
-    """
-    A source class that loads variables from a TOML file
-    """
-
-    def __init__(
-        self,
-        settings_cls: type[BaseSettings],
-        toml_file: Path | None=None,
-    ):
-        self.toml_file_path = toml_file or settings_cls.model_config.get("toml_file")
-        
-        self.nested_toml_data = self._read_files(self.toml_file_path)
-        self.toml_data = {}
-        for top_level_key, top_level_value in self.nested_toml_data.items():
-            if isinstance(top_level_value, dict):
-                # value is nested, flatten it
-                for key, value in top_level_value.items():
-                    self.toml_data[key] = value
-            else:
-                # value is already flat, just set it as-is
-                self.toml_data[top_level_key] = top_level_value
-                
-        # filter toml_data to only include keys that are defined on this settings_cls
-        self.toml_data = {
-            key: value
-            for key, value in self.toml_data.items()
-            if key in settings_cls.model_fields
-        }
-            
-        super(TomlConfigSettingsSource, self).__init__(settings_cls, self.toml_data)
-
-
-class ArchiveBoxBaseConfig(BaseSettings):
-    """
-    This is the base class for an ArchiveBox ConfigSet.
-    It handles loading values from schema defaults, ArchiveBox.conf TOML config, and environment variables.
-
-    class WgetConfig(ArchiveBoxBaseConfig):
-        WGET_BINARY: str = Field(default='wget', alias='WGET_BINARY_PATH')
-
-    c = WgetConfig()
-    print(c.WGET_BINARY)                    # outputs: wget
-
-    # you can mutate process environment variable and reload config using .__init__()
-    os.environ['WGET_BINARY_PATH'] = 'wget2'
-    c.__init__()
-
-    print(c.WGET_BINARY)                    # outputs: wget2
-
-    """
-    
-    # these pydantic config options are all VERY carefully chosen, make sure to test thoroughly before changing!!!
-    model_config = SettingsConfigDict(
-        validate_default=False,
-        case_sensitive=True,
-        extra="ignore",
-        arbitrary_types_allowed=False,
-        populate_by_name=True,
-        from_attributes=True,
-        loc_by_alias=False,
-        validate_assignment=True,
-        validate_return=True,
-        revalidate_instances="always",
-    )
-    
-    load_from_defaults: ClassVar[bool] = True
-    load_from_configfile: ClassVar[bool] = True
-    load_from_environment: ClassVar[bool] = True
-
-    @classmethod
-    def settings_customise_sources(
-        cls,
-        settings_cls: Type[BaseSettings],
-        init_settings: PydanticBaseSettingsSource,
-        env_settings: PydanticBaseSettingsSource,
-        dotenv_settings: PydanticBaseSettingsSource,
-        file_secret_settings: PydanticBaseSettingsSource,
-    ) -> Tuple[PydanticBaseSettingsSource, ...]:
-        """Defines the config precedence order: Schema defaults -> ArchiveBox.conf (TOML) -> Environment variables"""
-        
-        ARCHIVEBOX_CONFIG_FILE = DATA_DIR / "ArchiveBox.conf"
-        ARCHIVEBOX_CONFIG_FILE_BAK = ARCHIVEBOX_CONFIG_FILE.parent / ".ArchiveBox.conf.bak"
-        
-        # import ipdb; ipdb.set_trace()
-        
-        precedence_order = {}
-        
-        # if ArchiveBox.conf does not exist yet, return defaults -> env order
-        if not ARCHIVEBOX_CONFIG_FILE.is_file():
-            precedence_order = {
-                'defaults': init_settings,
-                'environment': env_settings,
-            }
-        
-        # if ArchiveBox.conf exists and is in TOML format, return default -> TOML -> env order
-        try:
-            precedence_order = precedence_order or {
-                'defaults': init_settings,
-                'configfile': FlatTomlConfigSettingsSource(settings_cls, toml_file=ARCHIVEBOX_CONFIG_FILE),
-                'environment': env_settings,
-            }
-        except Exception as err:
-            if err.__class__.__name__ != "TOMLDecodeError":
-                raise
-            # if ArchiveBox.conf exists and is in INI format, convert it then return default -> TOML -> env order
-
-            # Convert ArchiveBox.conf in INI format to TOML and save original to .ArchiveBox.bak
-            original_ini = ARCHIVEBOX_CONFIG_FILE.read_text()
-            ARCHIVEBOX_CONFIG_FILE_BAK.write_text(original_ini)
-            new_toml = toml_util.convert(original_ini)
-            ARCHIVEBOX_CONFIG_FILE.write_text(new_toml)
-
-            precedence_order = {
-                'defaults': init_settings,
-                'configfile': FlatTomlConfigSettingsSource(settings_cls, toml_file=ARCHIVEBOX_CONFIG_FILE),
-                'environment': env_settings,
-            }
-            
-        if not cls.load_from_environment:
-            precedence_order.pop('environment')
-        if not cls.load_from_configfile:
-            precedence_order.pop('configfile')
-        if not cls.load_from_defaults:
-            precedence_order.pop('defaults')
-
-        return tuple(precedence_order.values())
-
-    @model_validator(mode="after")
-    def fill_defaults(self):
-        """Populate any unset values using function provided as their default"""
-
-        for key, field in self.model_fields.items():
-            value = getattr(self, key)
-            
-            if isinstance(value, Callable):
-                # if value is a function, execute it to get the actual value, passing existing config as a dict arg if expected
-                if func_takes_args_or_kwargs(value):
-                    # assemble dict of existing field values to pass to default factory functions
-                    config_so_far = benedict(self.model_dump(include=set(self.model_fields.keys()), warnings=False))
-                    computed_default = field.default(config_so_far)
-                else:
-                    # otherwise it's a pure function with no args, just call it
-                    computed_default = field.default()
-
-                # coerce/check to make sure default factory return value matches type annotation
-                TypeAdapter(field.annotation).validate_python(computed_default)
-
-                # set generated default value as final validated value
-                setattr(self, key, computed_default)
-        return self
-    
-    def update_in_place(self, warn=True, **kwargs):
-        """
-        Update the config with new values. Use this sparingly! We should almost never be updating config at runtime.
-        Sets them in the environment so they propagate to spawned subprocesses / across future re-__init__()s and reload from environment
-
-        Example acceptable use case: user config says SEARCH_BACKEND_ENGINE=sonic but sonic_client pip library is not installed so we cannot use it.
-        SEARCH_BACKEND_CONFIG.update_in_place(SEARCH_BACKEND_ENGINE='ripgrep') can be used to reset it back to ripgrep so we can continue.
-        """
-        if warn:
-            print('[!] WARNING: Some of the provided user config values cannot be used, temporarily ignoring them:')
-        for key, value in kwargs.items():
-            os.environ[key] = str(value)
-            original_value = getattr(self, key)
-            if warn:
-                print(f'    {key}={original_value} -> {value}')
-        self.__init__()
-        return self
-    
-    def as_legacy_config_schema(self):
-        # shim for backwards compatibility with old config schema style
-        model_values = self.model_dump()
-        return benedict({
-            key: {'type': field.annotation, 'default': model_values[key]}
-            for key, field in self.model_fields.items()
-        })
-
-
-class BaseConfigSet(ArchiveBoxBaseConfig, BaseHook):      # type: ignore[type-arg]
-    hook_type: ClassVar[HookType] = 'CONFIG'
-
-    # @abx.hookimpl
-    # def ready(self, settings):
-    #    # reload config from environment, in case it's been changed by any other plugins
-    #    self.__init__()
-
-
-    @abx.hookimpl
-    def get_CONFIGS(self):
-        try:
-            return {self.id: self}
-        except Exception as e:
-            # raise Exception(f'Error computing CONFIGS for {type(self)}: {e.__class__.__name__}: {e}')
-            print(f'Error computing CONFIGS for {type(self)}: {e.__class__.__name__}: {e}')
-        return {}
-
-    @abx.hookimpl
-    def get_FLAT_CONFIG(self):
-        try:
-            return self.model_dump()
-        except Exception as e:
-            # raise Exception(f'Error computing FLAT_CONFIG for {type(self)}: {e.__class__.__name__}: {e}')
-            print(f'Error computing FLAT_CONFIG for {type(self)}: {e.__class__.__name__}: {e}')
-        return {}
@@ -1,222 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import json
-import os
-
-from typing import Optional, List, Literal, Annotated, Dict, Any, Tuple
-from typing_extensions import Self
-from pathlib import Path
-
-from pydantic import model_validator, AfterValidator
-from pydantic_pkgr import BinName
-from django.utils.functional import cached_property
-from django.utils import timezone
-
-import abx
-
-from .base_hook import BaseHook, HookType
-from .base_binary import BaseBinary
-
-
-def no_empty_args(args: List[str]) -> List[str]:
-    assert all(len(arg) for arg in args)
-    return args
-
-ExtractorName = Literal['wget', 'warc', 'media', 'singlefile'] | str
-
-HandlerFuncStr = Annotated[str, AfterValidator(lambda s: s.startswith('self.'))]
-CmdArgsList = Annotated[List[str] | Tuple[str, ...], AfterValidator(no_empty_args)]
-
-
-class BaseExtractor(BaseHook):
-    hook_type: HookType = 'EXTRACTOR'
-    
-    name: ExtractorName
-    binary: BinName
-
-    output_path_func: HandlerFuncStr = 'self.get_output_path'
-    should_extract_func: HandlerFuncStr = 'self.should_extract'
-    extract_func: HandlerFuncStr = 'self.extract'
-    exec_func: HandlerFuncStr = 'self.exec'
-
-    default_args: CmdArgsList = []
-    extra_args: CmdArgsList = []
-    args: Optional[CmdArgsList] = None
-
-    @model_validator(mode='after')
-    def validate_model(self) -> Self:
-        if self.args is None:
-            self.args = [*self.default_args, *self.extra_args]
-        return self
-
-
-    def get_output_path(self, snapshot) -> Path:
-        return Path(self.id.lower())
-
-    def should_extract(self, snapshot) -> bool:
-        try:
-            assert self.detect_installed_binary().version
-        except Exception:
-            raise
-            # could not load binary
-            return False
-        
-        # output_dir = self.get_output_path(snapshot)
-        # if output_dir.glob('*.*'):
-        #     return False
-        return True
-
-    @abx.hookimpl
-    def extract(self, snapshot_id: str) -> Dict[str, Any]:
-        from core.models import Snapshot
-        from archivebox import CONSTANTS
-        
-        snapshot = Snapshot.objects.get(id=snapshot_id)
-        
-        if not self.should_extract(snapshot):
-            return {}
-        
-        status = 'failed'
-        start_ts = timezone.now()
-        uplink = self.detect_network_interface()
-        installed_binary = self.detect_installed_binary()
-        machine = installed_binary.machine
-        assert uplink.machine == installed_binary.machine  # it would be *very* weird if this wasn't true
-        
-        # output_dir = self.get_output_path(snapshot) or CONSTANTS.TMP_DIR
-        output_dir = CONSTANTS.DATA_DIR / '.tmp' / 'extractors' / self.name / str(snapshot.abid)
-        output_dir.mkdir(parents=True, exist_ok=True)
-
-        # execute the extractor binary with the given args
-        args = [snapshot.url, *self.args] if self.args is not None else [snapshot.url, *self.default_args, *self.extra_args]
-        cmd = [str(installed_binary.abspath), *args]
-        proc = self.exec(installed_binary=installed_binary, args=args, cwd=output_dir)
-
-        # collect the output
-        end_ts = timezone.now()
-        output_files = list(str(path.relative_to(output_dir)) for path in output_dir.glob('**/*.*'))
-        stdout = proc.stdout.strip()
-        stderr = proc.stderr.strip()
-        output_json = None
-        output_text = stdout
-        try:
-            output_json = json.loads(stdout.strip())
-            output_text = None
-        except json.JSONDecodeError:
-            pass
-        
-        errors = []
-        if proc.returncode == 0:
-            status = 'success'
-        else:
-            errors.append(f'{installed_binary.name} returned non-zero exit code: {proc.returncode}')   
-
-        # increment health stats counters
-        if status == 'success':
-            machine.record_health_success()
-            uplink.record_health_success()
-            installed_binary.record_health_success()
-        else:
-            machine.record_health_failure()
-            uplink.record_health_failure()
-            installed_binary.record_health_failure()
-
-        return {
-            'extractor': self.name,
-            
-            'snapshot': {
-                'id': snapshot.id,
-                'abid': snapshot.abid,
-                'url': snapshot.url,
-                'created_by_id': snapshot.created_by_id,
-            },
-            
-            'machine': {
-                'id': machine.id,
-                'abid': machine.abid,
-                'guid': machine.guid,
-                'hostname': machine.hostname,
-                'hw_in_docker': machine.hw_in_docker,
-                'hw_in_vm': machine.hw_in_vm,
-                'hw_manufacturer': machine.hw_manufacturer,
-                'hw_product': machine.hw_product,
-                'hw_uuid': machine.hw_uuid,
-                'os_arch': machine.os_arch,
-                'os_family': machine.os_family,
-                'os_platform': machine.os_platform,
-                'os_release': machine.os_release,
-                'os_kernel': machine.os_kernel,
-            },
-            
-            'uplink': { 
-                'id': uplink.id,
-                'abid': uplink.abid,
-                'mac_address': uplink.mac_address,
-                'ip_public': uplink.ip_public,
-                'ip_local': uplink.ip_local,
-                'dns_server': uplink.dns_server,
-                'hostname': uplink.hostname,
-                'iface': uplink.iface,
-                'isp': uplink.isp,
-                'city': uplink.city,
-                'region': uplink.region,
-                'country': uplink.country,
-            },
-            
-            'binary': {
-                'id': installed_binary.id,
-                'abid': installed_binary.abid,
-                'name': installed_binary.name,
-                'binprovider': installed_binary.binprovider,
-                'abspath': installed_binary.abspath,
-                'version': installed_binary.version,
-                'sha256': installed_binary.sha256,
-            },
-
-            'cmd': cmd,
-            'stdout': stdout,
-            'stderr': stderr,
-            'returncode': proc.returncode,
-            'start_ts': start_ts,
-            'end_ts': end_ts,
-            
-            'status': status,
-            'errors': errors,
-            'output_dir': str(output_dir.relative_to(CONSTANTS.DATA_DIR)),
-            'output_files': output_files,
-            'output_json': output_json or {},
-            'output_text': output_text or '',
-        }
-
-    # TODO: move this to a hookimpl
-    def exec(self, args: CmdArgsList=(), cwd: Optional[Path]=None, installed_binary=None):
-        cwd = cwd or Path(os.getcwd())
-        binary = self.load_binary(installed_binary=installed_binary)
-        
-        return binary.exec(cmd=args, cwd=cwd)
-    
-    @cached_property
-    def BINARY(self) -> BaseBinary:
-        import abx.archivebox.use
-        for binary in abx.archivebox.use.get_BINARIES().values():
-            if binary.name == self.binary:
-                return binary
-        raise ValueError(f'Binary {self.binary} not found')
-    
-    def detect_installed_binary(self):
-        from machine.models import InstalledBinary
-        # hydrates binary from DB/cache if record of installed version is recent enough
-        # otherwise it finds it from scratch by detecting installed version/abspath/sha256 on host
-        return InstalledBinary.objects.get_from_db_or_cache(self.BINARY)
-
-    def load_binary(self, installed_binary=None) -> BaseBinary:
-        installed_binary = installed_binary or self.detect_installed_binary()
-        return installed_binary.load_from_db()
-    
-    def detect_network_interface(self):
-        from machine.models import NetworkInterface
-        return NetworkInterface.objects.current()
-
-    @abx.hookimpl
-    def get_EXTRACTORS(self):
-        return [self]
@@ -1,80 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import inspect
-from huey.api import TaskWrapper
-
-from pathlib import Path
-from typing import Tuple, Literal, ClassVar, get_args
-from pydantic import BaseModel, ConfigDict
-from django.utils.functional import cached_property
-
-import abx
-
-HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE', 'SEARCHBACKEND']
-hook_type_names: Tuple[HookType] = get_args(HookType)
-
-class BaseHook(BaseModel):
-    model_config = ConfigDict(
-        extra="allow",
-        arbitrary_types_allowed=True,
-        from_attributes=True,
-        populate_by_name=True,
-        validate_defaults=True,
-        validate_assignment=False,
-        revalidate_instances="subclass-instances",
-        ignored_types=(TaskWrapper, cached_property),
-    )
-    
-    hook_type: ClassVar[HookType]     # e.g. = 'CONFIG'
-    
-    # verbose_name: str = Field()
-    
-    _is_registered: bool = False
-    _is_ready: bool = False
-
-
-    @property
-    def id(self) -> str:
-        return self.__class__.__name__
-
-    @property
-    def hook_module(self) -> str:
-        """e.g. plugins_extractor.singlefile.apps.SinglefileConfigSet"""
-        return f'{self.__module__}.{self.__class__.__name__}'
-
-    @property
-    def hook_file(self) -> Path:
-        """e.g. plugins_extractor.singlefile.apps.SinglefileConfigSet"""
-        return Path(inspect.getfile(self.__class__))
-
-    @property
-    def plugin_module(self) -> str:
-        """e.g. plugins_extractor.singlefile"""
-        return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit(".apps.", 1)[0]
-
-    @property
-    def plugin_dir(self) -> Path:
-        return Path(inspect.getfile(self.__class__)).parent.resolve()
-    
-    @property
-    def admin_url(self) -> str:
-        # e.g. /admin/environment/config/LdapConfig/
-        return f"/admin/environment/{self.hook_type.lower()}/{self.id}/"
-
-
-    @abx.hookimpl
-    def register(self, settings):
-        """Called when django.apps.AppConfig.ready() is called"""
-        
-        # print("REGISTERED HOOK:", self.hook_module)
-        self._is_registered = True
-        
-
-    @abx.hookimpl
-    def ready(self):
-        """Called when django.apps.AppConfig.ready() is called"""
-        
-        assert self._is_registered, f"Tried to run {self.hook_module}.ready() but it was never registered!"
-       
-        # print("READY HOOK:", self.hook_module)
-        self._is_ready = True
@@ -1,154 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import abx
-import inspect
-from pathlib import Path
-
-from django.apps import AppConfig
-
-from typing import List, Type, Dict
-from typing_extensions import Self
-
-from pydantic import (
-    BaseModel,
-    ConfigDict,
-    Field,
-    model_validator,
-    InstanceOf,
-    computed_field,
-)
-from benedict import benedict
-
-from .base_hook import BaseHook, HookType
-
-class BasePlugin(BaseModel):
-    model_config = ConfigDict(
-        extra='forbid',
-        arbitrary_types_allowed=True,
-        populate_by_name=True,
-        from_attributes=True,
-        validate_defaults=False,
-        validate_assignment=False,
-        revalidate_instances="always",
-        # frozen=True,
-    )
-
-    # Required by AppConfig:
-    app_label: str = Field()                      # e.g. 'singlefile'                  (one-word machine-readable representation, to use as url-safe id/db-table prefix_/attr name)
-    verbose_name: str = Field()                   # e.g. 'SingleFile'                  (human-readable *short* label, for use in column names, form labels, etc.)
-    docs_url: str = Field(default=None)           # e.g. 'https://github.com/...'
-    
-    # All the hooks the plugin will install:
-    hooks: List[InstanceOf[BaseHook]] = Field(default=[])
-    
-    _is_registered: bool = False
-    _is_ready: bool = False
-    
-    @computed_field
-    @property
-    def id(self) -> str:
-        return self.__class__.__name__
-    
-    @property
-    def name(self) -> str:
-        return self.app_label
-    
-    # @computed_field
-    @property
-    def plugin_module(self) -> str:  # DottedImportPath
-        """ "
-        Dotted import path of the plugin's module (after its loaded via settings.INSTALLED_APPS).
-        e.g. 'archivebox.plugins_pkg.npm.apps.NpmPlugin' -> 'plugins_pkg.npm'
-        """
-        return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit('.apps.', 1)[0]
-
-
-    @property
-    def plugin_module_full(self) -> str:  # DottedImportPath
-        """e.g. 'archivebox.plugins_pkg.npm.apps.NpmPlugin'"""
-        return f"{self.__module__}.{self.__class__.__name__}"
-    
-    # @computed_field
-    @property
-    def plugin_dir(self) -> Path:
-        return Path(inspect.getfile(self.__class__)).parent.resolve()
-    
-    @model_validator(mode='after')
-    def validate(self) -> Self:
-        """Validate the plugin's build-time configuration here before it's registered in Django at runtime."""
-        
-        # VERY IMPORTANT:
-        # preserve references to original default objects,
-        # pydantic deepcopies them by default which breaks mutability
-        # see https://github.com/pydantic/pydantic/issues/7608
-        # if we dont do this, then plugins_extractor.SINGLEFILE_CONFIG != settings.CONFIGS.SingleFileConfig for example
-        # and calling .__init__() on one of them will not update the other
-        self.hooks = self.model_fields['hooks'].default
-        
-        assert self.app_label and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name'
-        
-        # assert json.dumps(self.model_json_schema(), indent=4), f"Plugin {self.plugin_module} has invalid JSON schema."
-        
-        return self
-    
-    @property
-    def AppConfig(plugin_self) -> Type[AppConfig]:
-        """Generate a Django AppConfig class for this plugin."""
-
-
-        class PluginAppConfig(AppConfig):
-            """Django AppConfig for plugin, allows it to be loaded as a Django app listed in settings.INSTALLED_APPS."""
-            name = plugin_self.plugin_module
-            app_label = plugin_self.app_label
-            verbose_name = plugin_self.verbose_name
-
-            default_auto_field = 'django.db.models.AutoField'
-
-            # handled by abx.hookimpl  ready()
-            # def ready(self):
-            #     from django.conf import settings
-            #     plugin_self.ready(settings)
-
-        return PluginAppConfig
-
-    @property
-    def HOOKS_BY_ID(self) -> Dict[str, InstanceOf[BaseHook]]:
-        return benedict({hook.id: hook for hook in self.hooks})
-
-    @property
-    def HOOKS_BY_TYPE(self) -> Dict[HookType, Dict[str, InstanceOf[BaseHook]]]:
-        hooks = benedict({})
-        for hook in self.hooks:
-            hooks[hook.hook_type] = hooks.get(hook.hook_type) or benedict({})
-            hooks[hook.hook_type][hook.id] = hook
-        return hooks
-
-
-
-    @abx.hookimpl
-    def register(self, settings):
-        from archivebox.config.legacy import bump_startup_progress_bar
-
-        self._is_registered = True
-        bump_startup_progress_bar()
-
-        # print('◣----------------- REGISTERED PLUGIN:', self.plugin_module, '-----------------◢')
-        # print()
-
-    @abx.hookimpl
-    def ready(self, settings=None):
-        """Runs any runtime code needed when AppConfig.ready() is called (after all models are imported)."""
-
-        from archivebox.config.legacy import bump_startup_progress_bar
-
-        assert self._is_registered, f"Tried to run {self.plugin_module}.ready() but it was never registered!"
-        self._is_ready = True
-
-        # settings.PLUGINS[self.id]._is_ready = True
-        bump_startup_progress_bar()
-
-
-    @abx.hookimpl
-    def get_INSTALLED_APPS(self):
-        return [self.plugin_module]
-
@@ -1,106 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import importlib
-
-from typing import Dict, List, TYPE_CHECKING
-from pydantic import Field, InstanceOf
-from benedict import benedict
-
-if TYPE_CHECKING:
-    from huey.api import TaskWrapper
-
-import abx
-
-from .base_hook import BaseHook, HookType
-from .base_binary import BaseBinary
-
-
-
-class BaseQueue(BaseHook):
-    hook_type: HookType = 'QUEUE'
-
-    name: str = Field()       # e.g. 'singlefile'
-
-    binaries: List[InstanceOf[BaseBinary]] = Field()
-
-    @property
-    def tasks(self) -> Dict[str, 'TaskWrapper']:
-        """Return an dict of all the background worker tasks defined in the plugin's tasks.py file."""
-        tasks = importlib.import_module(f"{self.plugin_module}.tasks")
-
-        all_tasks = {}
-
-        for task_name, task in tasks.__dict__.items():
-            # if attr is a Huey task and its queue_name matches our hook's queue name
-            if hasattr(task, "task_class") and task.huey.name == self.name:
-                all_tasks[task_name] = task
-
-        return benedict(all_tasks)
-
-    def get_django_huey_config(self, QUEUE_DATABASE_NAME) -> dict:
-        """Get the config dict to insert into django.conf.settings.DJANGO_HUEY['queues']."""
-        return {
-            "huey_class": "huey.SqliteHuey",
-            "filename": QUEUE_DATABASE_NAME,
-            "name": self.name,
-            "results": True,
-            "store_none": True,
-            "immediate": False,
-            "utc": True,
-            "consumer": {
-                "workers": 1,
-                "worker_type": "thread",
-                "initial_delay": 0.1,  # Smallest polling interval, same as -d.
-                "backoff": 1.15,  # Exponential backoff using this rate, -b.
-                "max_delay": 10.0,  # Max possible polling interval, -m.
-                "scheduler_interval": 1,  # Check schedule every second, -s.
-                "periodic": True,  # Enable crontab feature.
-                "check_worker_health": True,  # Enable worker health checks.
-                "health_check_interval": 1,  # Check worker health every second.
-            },
-        }
-        
-    def get_supervisord_config(self, settings) -> dict:
-        """Ge the config dict used to tell sueprvisord to start a huey consumer for this queue."""
-        return {
-            "name": f"worker_{self.name}",
-            "command": f"archivebox manage djangohuey --queue {self.name}",
-            "stdout_logfile": f"logs/worker_{self.name}.log",
-            "redirect_stderr": "true",
-            "autorestart": "true",
-            "autostart": "false",
-        }
-        
-    def start_supervisord_worker(self, settings, lazy=True):
-        from queues.supervisor_util import get_or_create_supervisord_process, start_worker
-        print()
-        try:
-            supervisor = get_or_create_supervisord_process(daemonize=False)
-        except Exception as e:
-            print(f"Error starting worker for queue {self.name}: {e}")
-            return None
-        print()
-        worker = start_worker(supervisor, self.get_supervisord_config(settings), lazy=lazy)
-
-        # Update settings.WORKERS to include this worker
-        settings.WORKERS = getattr(settings, "WORKERS", None) or benedict({})
-        settings.WORKERS[self.id] = self.start_supervisord_worker(settings, lazy=True)
-
-        return worker
-
-    @abx.hookimpl
-    def get_QUEUES(self):
-        return [self]
-
-    @abx.hookimpl
-    def get_DJANGO_HUEY_QUEUES(self, QUEUE_DATABASE_NAME):
-        """queue configs to be added to django.conf.settings.DJANGO_HUEY['queues']"""
-        return {
-            self.name: self.get_django_huey_config(QUEUE_DATABASE_NAME)
-        }
-        
-        
-    # @abx.hookimpl
-    # def ready(self, settings):
-    #     self.start_supervisord_worker(settings, lazy=True)
-    #     super().ready(settings)
@@ -1,29 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import abx
-
-from .base_hook import BaseHook, HookType
-
-
-class BaseReplayer(BaseHook):
-    """Describes how to render an ArchiveResult in several contexts"""
-    
-    hook_type: HookType = 'REPLAYER'
-    
-    url_pattern: str = '*'
-
-    row_template: str = 'plugins/generic_replayer/templates/row.html'
-    embed_template: str = 'plugins/generic_replayer/templates/embed.html'
-    fullpage_template: str = 'plugins/generic_replayer/templates/fullpage.html'
-
-    # row_view: LazyImportStr = 'plugins.generic_replayer.views.row_view'
-    # embed_view: LazyImportStr = 'plugins.generic_replayer.views.embed_view'
-    # fullpage_view: LazyImportStr = 'plugins.generic_replayer.views.fullpage_view'
-    # icon_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
-    # thumbnail_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
-
-    @abx.hookimpl
-    def get_REPLAYERS(self):
-        return [self]
-
-    # TODO: add hookimpl methods for get_row_template, get_embed_template, get_fullpage_template, etc...
@@ -1,33 +0,0 @@
-__package__ = 'abx.archivebox'
-
-from typing import Iterable, List
-from pydantic import Field
-
-import abx
-from .base_hook import BaseHook, HookType
-
-
-
-class BaseSearchBackend(BaseHook):
-    hook_type: HookType = 'SEARCHBACKEND'
-
-    name: str = Field()       # e.g. 'singlefile'
-
-
-    # TODO: move these to a hookimpl
-
-    @staticmethod
-    def index(snapshot_id: str, texts: List[str]):
-        return
-
-    @staticmethod
-    def flush(snapshot_ids: Iterable[str]):
-        return
-
-    @staticmethod
-    def search(text: str) -> List[str]:
-        raise NotImplementedError("search method must be implemented by subclass")
-    
-    @abx.hookimpl
-    def get_SEARCHBACKENDS(self):
-        return [self]
@@ -1,39 +0,0 @@
-__package__ = 'abx.archivebox'
-
-from typing import Dict, Any
-
-from .. import hookspec
-
-
-@hookspec
-def get_CONFIGS():
-    return {}
-
-@hookspec
-def get_EXTRACTORS():
-    return {}
-
-@hookspec
-def get_REPLAYERS():
-    return {}
-
-@hookspec
-def get_CHECKS():
-    return {}
-
-@hookspec
-def get_ADMINDATAVIEWS():
-    return {}
-
-@hookspec
-def get_QUEUES():
-    return {}
-
-@hookspec
-def get_SEARCHBACKENDS():
-    return {}
-
-
-@hookspec
-def extract(snapshot_id) -> Dict[str, Any]:
-    return {}
@@ -1,114 +0,0 @@
-from typing import Any, List, Callable
-
-import json
-import ast
-import inspect
-import toml
-import re
-import configparser
-
-from pathlib import Path, PosixPath
-
-from pydantic.json_schema import GenerateJsonSchema
-from pydantic_core import to_jsonable_python
-
-JSONValue = str | bool | int | None | List['JSONValue']
-
-TOML_HEADER = "# Converted from INI to TOML format: https://toml.io/en/\n\n"
-
-def load_ini_value(val: str) -> JSONValue:
-    """Convert lax INI values into strict TOML-compliant (JSON) values"""
-    if val.lower() in ('true', 'yes', '1'):
-        return True
-    if val.lower() in ('false', 'no', '0'):
-        return False
-    if val.isdigit():
-        return int(val)
-
-    try:
-        return ast.literal_eval(val)
-    except Exception:
-        pass
-
-    try:
-        return json.loads(val)
-    except Exception:
-        pass
-    
-    return val
-
-
-def convert(ini_str: str) -> str:
-    """Convert a string of INI config into its TOML equivalent (warning: strips comments)"""
-
-    config = configparser.ConfigParser()
-    config.optionxform = str  # capitalize key names
-    config.read_string(ini_str)
-
-    # Initialize an empty dictionary to store the TOML representation
-    toml_dict = {}
-
-    # Iterate over each section in the INI configuration
-    for section in config.sections():
-        toml_dict[section] = {}
-
-        # Iterate over each key-value pair in the section
-        for key, value in config.items(section):
-            parsed_value = load_ini_value(value)
-
-            # Convert the parsed value to its TOML-compatible JSON representation
-            toml_dict[section.upper()][key.upper()] = json.dumps(parsed_value)
-
-    # Build the TOML string
-    toml_str = TOML_HEADER
-    for section, items in toml_dict.items():
-        toml_str += f"[{section}]\n"
-        for key, value in items.items():
-            toml_str += f"{key} = {value}\n"
-        toml_str += "\n"
-
-    return toml_str.strip()
-
-
-
-class JSONSchemaWithLambdas(GenerateJsonSchema):
-    """
-    Encode lambda functions in default values properly.
-    Usage:
-    >>> json.dumps(value, encoder=JSONSchemaWithLambdas())
-    """
-    def encode_default(self, default: Any) -> Any:
-        config = self._config
-        if isinstance(default, Callable):
-            return '{{lambda ' + inspect.getsource(default).split('=lambda ')[-1].strip()[:-1] + '}}'
-        return to_jsonable_python(
-           default,
-           timedelta_mode=config.ser_json_timedelta,
-           bytes_mode=config.ser_json_bytes,
-           serialize_unknown=True
-        )
-
-    # for computed_field properties render them like this instead:
-    # inspect.getsource(field.wrapped_property.fget).split('def ', 1)[-1].split('\n', 1)[-1].strip().strip('return '),
-
-
-def better_toml_dump_str(val: Any) -> str:
-    try:
-        return toml.encoder._dump_str(val)     # type: ignore
-    except Exception:
-        # if we hit any of toml's numerous encoding bugs,
-        # fall back to using json representation of string
-        return json.dumps(str(val))
-
-class CustomTOMLEncoder(toml.encoder.TomlEncoder):
-    """
-    Custom TomlEncoder to work around https://github.com/uiri/toml's many encoding bugs.
-    More info: https://github.com/fabiocaccamo/python-benedict/issues/439
-    >>> toml.dumps(value, encoder=CustomTOMLEncoder())
-    """
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.dump_funcs[Path] = lambda x: json.dumps(str(x))
-        self.dump_funcs[PosixPath] = lambda x: json.dumps(str(x))
-        self.dump_funcs[str] = better_toml_dump_str
-        self.dump_funcs[re.RegexFlag] = better_toml_dump_str
@@ -1,138 +0,0 @@
-__package__ = 'abx.archivebox'
-
-from typing import Dict, Any, TYPE_CHECKING
-
-from django.utils import timezone
-from benedict import benedict
-
-from .. import pm
-
-if TYPE_CHECKING:
-    from .base_hook import BaseHook
-    from .base_configset import BaseConfigSet
-    from .base_binary import BaseBinary, BaseBinProvider
-    from .base_extractor import BaseExtractor
-    from .base_replayer import BaseReplayer
-    from .base_check import BaseCheck
-    from .base_queue import BaseQueue
-    from .base_admindataview import BaseAdminDataView
-    from .base_searchbackend import BaseSearchBackend
-
-# API exposed to ArchiveBox code
-
-def get_PLUGINS():
-    return benedict({
-        plugin.PLUGIN.id: plugin.PLUGIN
-        for plugin in pm.get_plugins()
-    })
-
-def get_HOOKS(PLUGINS) -> Dict[str, 'BaseHook']:
-    return benedict({
-        hook.id: hook
-        for plugin in PLUGINS.values()
-            for hook in plugin.hooks
-    })
-
-def get_CONFIGS() -> Dict[str, 'BaseConfigSet']:
-    return benedict({
-        config_id: config
-        for plugin_configs in pm.hook.get_CONFIGS()
-            for config_id, config in plugin_configs.items()
-    })
-    
-def get_FLAT_CONFIG() -> Dict[str, Any]:
-    return benedict({
-        key: value
-        for plugin_config_dict in pm.hook.get_FLAT_CONFIG()
-            for key, value in plugin_config_dict.items()
-    })
-
-def get_BINPROVIDERS() -> Dict[str, 'BaseBinProvider']:
-    # TODO: move these to plugins
-    from abx.archivebox.base_binary import apt, brew, env
-    builtin_binproviders = [apt, brew, env]
-    
-    return benedict({
-        binprovider.id: binprovider
-        for plugin_binproviders in [builtin_binproviders, *pm.hook.get_BINPROVIDERS()]
-            for binprovider in plugin_binproviders
-    })
-
-def get_BINARIES() -> Dict[str, 'BaseBinary']:
-    return benedict({
-        binary.id: binary
-        for plugin_binaries in pm.hook.get_BINARIES()
-            for binary in plugin_binaries
-    })
-
-def get_EXTRACTORS() -> Dict[str, 'BaseExtractor']:
-    return benedict({
-        extractor.id: extractor
-        for plugin_extractors in pm.hook.get_EXTRACTORS()
-            for extractor in plugin_extractors
-    })
-
-def get_REPLAYERS() -> Dict[str, 'BaseReplayer']:
-    return benedict({
-        replayer.id: replayer
-        for plugin_replayers in pm.hook.get_REPLAYERS()
-            for replayer in plugin_replayers
-    })
-
-def get_CHECKS() -> Dict[str, 'BaseCheck']:
-    return benedict({
-        check.id: check
-        for plugin_checks in pm.hook.get_CHECKS()
-            for check in plugin_checks
-    })
-
-def get_ADMINDATAVIEWS() -> Dict[str, 'BaseAdminDataView']:
-    return benedict({
-        admin_dataview.id: admin_dataview
-        for plugin_admin_dataviews in pm.hook.get_ADMINDATAVIEWS()
-            for admin_dataview in plugin_admin_dataviews
-    })
-
-def get_QUEUES() -> Dict[str, 'BaseQueue']:
-    return benedict({
-        queue.id: queue
-        for plugin_queues in pm.hook.get_QUEUES()
-            for queue in plugin_queues
-    })
-
-def get_SEARCHBACKENDS() -> Dict[str, 'BaseSearchBackend']:
-    return benedict({
-        searchbackend.id: searchbackend
-        for plugin_searchbackends in pm.hook.get_SEARCHBACKENDS()
-            for searchbackend in plugin_searchbackends
-    })
-
-
-###########################
-
-
-def register_all_hooks(settings):
-    pm.hook.register(settings=settings)
-
-
-
-def extract(url_or_snapshot_id):
-    from core.models import Snapshot
-    
-    url, snapshot_abid, snapshot_id = None, None, None
-    snapshot = None
-    if '://' in url_or_snapshot_id:
-        url = url_or_snapshot_id
-        try:
-            snapshot = Snapshot.objects.get(url=url)
-        except Snapshot.DoesNotExist:
-            snapshot = Snapshot(url=url_or_snapshot_id, timestamp=str(timezone.now().timestamp()), bookmarked_at=timezone.now())
-            snapshot.save()
-    elif '-' in url_or_snapshot_id:
-        snapshot_id = url_or_snapshot_id
-        snapshot = Snapshot.objects.get(id=snapshot_id)
-    else:
-        snapshot_abid = url_or_snapshot_id
-        snapshot = Snapshot.objects.get(abid=snapshot_abid)
-
-    return pm.hook.extract(snapshot_id=snapshot.id)
@@ -1 +0,0 @@
-__package__ = 'abx.django'
@@ -1,13 +0,0 @@
-__package__ = 'abx.django'
-
-from django.apps import AppConfig
-
-
-class ABXConfig(AppConfig):
-    name = 'abx'
-
-    def ready(self):
-        import abx
-        from django.conf import settings
-        
-        abx.pm.hook.ready(settings=settings)
@@ -1,120 +0,0 @@
-__package__ = 'abx.django'
-
-from ..hookspec import hookspec
-
-
-###########################################################################################
-
-@hookspec
-def get_INSTALLED_APPS():
-    """Return a list of apps to add to INSTALLED_APPS"""
-    # e.g. ['your_plugin_type.plugin_name']
-    return []
-
-# @hookspec
-# def register_INSTALLED_APPS(INSTALLED_APPS):
-#     """Mutate INSTALLED_APPS in place to add your app in a specific position"""
-#     # idx_of_contrib = INSTALLED_APPS.index('django.contrib.auth')
-#     # INSTALLED_APPS.insert(idx_of_contrib + 1, 'your_plugin_type.plugin_name')
-#     pass
-
-
-@hookspec
-def get_TEMPLATE_DIRS():
-    return []     # e.g. ['your_plugin_type/plugin_name/templates']
-
-# @hookspec
-# def register_TEMPLATE_DIRS(TEMPLATE_DIRS):
-#     """Install django settings"""
-#     # e.g. TEMPLATE_DIRS.insert(0, 'your_plugin_type/plugin_name/templates')
-#     pass
-
-
-@hookspec
-def get_STATICFILES_DIRS():
-    return []     # e.g. ['your_plugin_type/plugin_name/static']
-
-# @hookspec
-# def register_STATICFILES_DIRS(STATICFILES_DIRS):
-#     """Mutate STATICFILES_DIRS in place to add your static dirs in a specific position"""
-#     # e.g. STATICFILES_DIRS.insert(0, 'your_plugin_type/plugin_name/static')
-#     pass
-
-
-@hookspec
-def get_MIDDLEWARE():
-    return []     # e.g. ['your_plugin_type.plugin_name.middleware.YourMiddleware']
-
-# @hookspec
-# def register_MIDDLEWARE(MIDDLEWARE):
-#     """Mutate MIDDLEWARE in place to add your middleware in a specific position"""
-#     # e.g. MIDDLEWARE.insert(0, 'your_plugin_type.plugin_name.middleware.YourMiddleware')
-#     pass
-
-
-@hookspec
-def get_AUTHENTICATION_BACKENDS():
-    return []     # e.g. ['django_auth_ldap.backend.LDAPBackend']
-
-# @hookspec
-# def register_AUTHENTICATION_BACKENDS(AUTHENTICATION_BACKENDS):
-#     """Mutate AUTHENTICATION_BACKENDS in place to add your auth backends in a specific position"""
-#     # e.g. AUTHENTICATION_BACKENDS.insert(0, 'your_plugin_type.plugin_name.backend.YourBackend')
-#     pass
-
-@hookspec
-def get_DJANGO_HUEY_QUEUES(QUEUE_DATABASE_NAME):
-    return []     # e.g. [{'name': 'your_plugin_type.plugin_name', 'HUEY': {...}}]
-
-# @hookspec
-# def register_DJANGO_HUEY(DJANGO_HUEY):
-#     """Mutate DJANGO_HUEY in place to add your huey queues in a specific position"""
-#     # e.g. DJANGO_HUEY['queues']['some_queue_name']['some_setting'] = 'some_value'
-#     pass
-
-
-@hookspec
-def get_ADMIN_DATA_VIEWS_URLS():
-    return []
-
-# @hookspec
-# def register_ADMIN_DATA_VIEWS(ADMIN_DATA_VIEWS):
-#     """Mutate ADMIN_DATA_VIEWS in place to add your admin data views in a specific position"""
-#     # e.g. ADMIN_DATA_VIEWS['URLS'].insert(0, 'your_plugin_type/plugin_name/admin_data_views.py')
-#     pass
-
-
-# @hookspec
-# def register_settings(settings):
-#     """Mutate settings in place to add your settings / modify existing settings"""
-#     # settings.SOME_KEY = 'some_value'
-#     pass
-
-
-###########################################################################################
-
-@hookspec
-def get_urlpatterns():
-    return []     # e.g. [path('your_plugin_type/plugin_name/url.py', your_view)]
-
-# @hookspec
-# def register_urlpatterns(urlpatterns):
-#     """Mutate urlpatterns in place to add your urlpatterns in a specific position"""
-#     # e.g. urlpatterns.insert(0, path('your_plugin_type/plugin_name/url.py', your_view))
-#     pass
-
-###########################################################################################
-
-@hookspec
-def register_checks():
-    """Register django checks with django system checks system"""
-    pass
-
-
-###########################################################################################
-
-
-@hookspec
-def ready():
-    """Called when Django apps app.ready() are triggered"""
-    pass
@@ -1,98 +0,0 @@
-__package__ = 'abx.django'
-
-import itertools
-# from benedict import benedict
-
-from .. import pm
-
-
-def get_INSTALLED_APPS():
-    return itertools.chain(*reversed(pm.hook.get_INSTALLED_APPS()))
-
-# def register_INSTALLLED_APPS(INSTALLED_APPS):
-#     pm.hook.register_INSTALLED_APPS(INSTALLED_APPS=INSTALLED_APPS)
-
-
-def get_MIDDLEWARES():
-    return itertools.chain(*reversed(pm.hook.get_MIDDLEWARE()))
-
-# def register_MIDDLEWARES(MIDDLEWARE):
-#     pm.hook.register_MIDDLEWARE(MIDDLEWARE=MIDDLEWARE)
-
-
-def get_AUTHENTICATION_BACKENDS():
-    return itertools.chain(*reversed(pm.hook.get_AUTHENTICATION_BACKENDS()))
-
-# def register_AUTHENTICATION_BACKENDS(AUTHENTICATION_BACKENDS):
-#     pm.hook.register_AUTHENTICATION_BACKENDS(AUTHENTICATION_BACKENDS=AUTHENTICATION_BACKENDS)
-
-
-def get_STATICFILES_DIRS():
-    return itertools.chain(*reversed(pm.hook.get_STATICFILES_DIRS()))
-
-# def register_STATICFILES_DIRS(STATICFILES_DIRS):
-#     pm.hook.register_STATICFILES_DIRS(STATICFILES_DIRS=STATICFILES_DIRS)
-
-
-def get_TEMPLATE_DIRS():
-    return itertools.chain(*reversed(pm.hook.get_TEMPLATE_DIRS()))
-
-# def register_TEMPLATE_DIRS(TEMPLATE_DIRS):
-#     pm.hook.register_TEMPLATE_DIRS(TEMPLATE_DIRS=TEMPLATE_DIRS)
-
-def get_DJANGO_HUEY_QUEUES(QUEUE_DATABASE_NAME='queue.sqlite3'):
-    HUEY_QUEUES = {}
-    for plugin_result in pm.hook.get_DJANGO_HUEY_QUEUES(QUEUE_DATABASE_NAME=QUEUE_DATABASE_NAME):
-        HUEY_QUEUES.update(plugin_result)
-    return HUEY_QUEUES
-
-# def register_DJANGO_HUEY(DJANGO_HUEY):
-#     pm.hook.register_DJANGO_HUEY(DJANGO_HUEY=DJANGO_HUEY)
-
-def get_ADMIN_DATA_VIEWS_URLS():
-    return itertools.chain(*reversed(pm.hook.get_ADMIN_DATA_VIEWS_URLS()))
-
-# def register_ADMIN_DATA_VIEWS(ADMIN_DATA_VIEWS):
-#     pm.hook.register_ADMIN_DATA_VIEWS(ADMIN_DATA_VIEWS=ADMIN_DATA_VIEWS)
-
-
-# def register_settings(settings):
-#     # convert settings dict to an benedict so we can set values using settings.attr = xyz notation
-#     settings_as_obj = benedict(settings, keypath_separator=None)
-    
-#     # set default values for settings that are used by plugins
-#     # settings_as_obj.INSTALLED_APPS = settings_as_obj.get('INSTALLED_APPS', [])
-#     # settings_as_obj.MIDDLEWARE = settings_as_obj.get('MIDDLEWARE', [])
-#     # settings_as_obj.AUTHENTICATION_BACKENDS = settings_as_obj.get('AUTHENTICATION_BACKENDS', [])
-#     # settings_as_obj.STATICFILES_DIRS = settings_as_obj.get('STATICFILES_DIRS', [])
-#     # settings_as_obj.TEMPLATE_DIRS = settings_as_obj.get('TEMPLATE_DIRS', [])
-#     # settings_as_obj.DJANGO_HUEY = settings_as_obj.get('DJANGO_HUEY', {'queues': {}})
-#     # settings_as_obj.ADMIN_DATA_VIEWS = settings_as_obj.get('ADMIN_DATA_VIEWS', {'URLS': []})
-    
-#     # # call all the hook functions to mutate the settings values in-place
-#     # register_INSTALLLED_APPS(settings_as_obj.INSTALLED_APPS)
-#     # register_MIDDLEWARES(settings_as_obj.MIDDLEWARE)
-#     # register_AUTHENTICATION_BACKENDS(settings_as_obj.AUTHENTICATION_BACKENDS)
-#     # register_STATICFILES_DIRS(settings_as_obj.STATICFILES_DIRS)
-#     # register_TEMPLATE_DIRS(settings_as_obj.TEMPLATE_DIRS)
-#     # register_DJANGO_HUEY(settings_as_obj.DJANGO_HUEY)
-#     # register_ADMIN_DATA_VIEWS(settings_as_obj.ADMIN_DATA_VIEWS)
-    
-#     # calls Plugin.settings(settings) on each registered plugin
-#     pm.hook.register_settings(settings=settings_as_obj)
-    
-#     # then finally update the settings globals() object will all the new settings
-#     # settings.update(settings_as_obj)
-
-
-def get_urlpatterns():
-    return list(itertools.chain(*pm.hook.urlpatterns()))
-
-def register_urlpatterns(urlpatterns):
-    pm.hook.register_urlpatterns(urlpatterns=urlpatterns)
-
-
-def register_checks():
-    """register any django system checks"""
-    pm.hook.register_checks()
-
@@ -1,14 +0,0 @@
-from pathlib import Path
-
-from pluggy import HookimplMarker
-from pluggy import HookspecMarker
-
-spec = hookspec = HookspecMarker("abx")
-impl = hookimpl = HookimplMarker("abx")
-
-
-@hookspec
-@hookimpl
-def get_system_user() -> str:
-    return Path('~').expanduser().name
-
@@ -1,30 +0,0 @@
-import inspect
-
-import pluggy
-
-
-class PluginManager(pluggy.PluginManager):
-    """
-    Patch to fix pluggy's PluginManager to work with pydantic models.
-    See: https://github.com/pytest-dev/pluggy/pull/536
-    """
-    def parse_hookimpl_opts(self, plugin, name: str) -> pluggy.HookimplOpts | None:
-        # IMPORTANT: @property methods can have side effects, and are never hookimpl
-        # if attr is a property, skip it in advance
-        plugin_class = plugin if inspect.isclass(plugin) else type(plugin)
-        if isinstance(getattr(plugin_class, name, None), property):
-            return None
-
-        # pydantic model fields are like attrs and also can never be hookimpls
-        plugin_is_pydantic_obj = hasattr(plugin, "__pydantic_core_schema__")
-        if plugin_is_pydantic_obj and name in getattr(plugin, "model_fields", {}):
-            # pydantic models mess with the class and attr __signature__
-            # so inspect.isroutine(...) throws exceptions and cant be used
-            return None
-        
-        try:
-            return super().parse_hookimpl_opts(plugin, name)
-        except AttributeError:
-            return super().parse_hookimpl_opts(type(plugin), name)
-
-pm = PluginManager("abx")
@@ -1 +0,0 @@
-__package__ = 'abx.pydantic_pkgr'
@@ -1,13 +0,0 @@
-
-from ..hookspec import hookspec
-
-###########################################################################################
-
-@hookspec
-def get_BINPROVIDERS():
-    return {}
-
-@hookspec
-def get_BINARIES():
-    return {}
-
@@ -1 +1 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"
@@ -0,0 +1,120 @@
+__package__ = "archivebox.api"
+
+from django import forms
+from django.contrib import admin
+from django.http import HttpRequest
+from django.utils.text import capfirst
+from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
+from signal_webhooks.settings import webhook_settings
+from signal_webhooks.utils import get_webhook_model, model_from_reference
+
+from archivebox.base_models.admin import BaseModelAdmin
+
+from archivebox.api.models import APIToken
+
+
+def _webhook_fields(*names: str) -> tuple[str, ...]:
+    model_fields = {field.name for field in get_webhook_model()._meta.fields}
+    return tuple(name for name in names if name in model_fields)
+
+
+class APITokenAdmin(BaseModelAdmin):
+    list_display = ("created_at", "id", "created_by", "token_redacted", "expires")
+    sort_fields = ("id", "created_at", "created_by", "expires")
+    readonly_fields = ("created_at", "modified_at")
+    search_fields = ("id", "created_by__username", "token")
+
+    fieldsets = (
+        (
+            "Token",
+            {
+                "fields": ("token", "expires"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Owner",
+            {
+                "fields": ("created_by",),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": ("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
+    )
+
+    list_filter = ("created_by",)
+    ordering = ["-created_at"]
+    list_per_page = 100
+
+
+class OutboundWebhookAdminForm(WebhookModelForm):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields["ref"] = forms.ChoiceField(
+            label=self.fields["ref"].label,
+            help_text=self.fields["ref"].help_text,
+            choices=[
+                (ref, f"{capfirst(model_from_reference(ref, check_hooks=False)._meta.verbose_name_plural)} ({ref})")
+                for ref in sorted(webhook_settings.HOOKS)
+            ],
+        )
+
+
+class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
+    form = OutboundWebhookAdminForm
+    list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
+    sort_fields = _webhook_fields("created_at", "created_by", "id", "ref", "endpoint", "last_success", "last_failure")
+    readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
+
+    fieldsets = (
+        (
+            "Webhook",
+            {
+                "fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"),
+                "classes": ("card", "wide"),
+            },
+        ),
+        (
+            "Authentication",
+            {
+                "fields": _webhook_fields("auth_token"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Status",
+            {
+                "fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Owner",
+            {
+                "fields": _webhook_fields("created_by"),
+                "classes": ("card",),
+            },
+        ),
+        (
+            "Timestamps",
+            {
+                "fields": _webhook_fields("created_at", "modified_at"),
+                "classes": ("card",),
+            },
+        ),
+    )
+
+    def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:
+        """Preserve WebhookAdmin's auth token filter with Django's current admin signature."""
+        return not lookup.startswith("auth_token") and admin.ModelAdmin.lookup_allowed(self, lookup, value, request)
+
+
+def register_admin(admin_site: admin.AdminSite) -> None:
+    admin_site.register(APIToken, APITokenAdmin)
+    admin_site.register(get_webhook_model(), CustomWebhookAdmin)
@@ -1,11 +1,14 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

 from django.apps import AppConfig


-
 class APIConfig(AppConfig):
-    name = 'api'
+    name = "archivebox.api"
+    label = "api"

-    def ready(self):
-        pass
+
+def register_admin(admin_site):
+    from archivebox.api.admin import register_admin
+
+    register_admin(admin_site)
@@ -1,21 +1,19 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

-from typing import Any, Optional, cast
 from datetime import timedelta

-from django.http import HttpRequest
 from django.utils import timezone
-from django.contrib.auth import login
+from django.http import HttpRequest
 from django.contrib.auth import authenticate
-from django.contrib.auth.models import AbstractBaseUser
+from django.contrib.auth.models import User

-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser
+from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader
 from ninja.errors import HttpError


-def get_or_create_api_token(user):
-    from api.models import APIToken
-    
+def get_or_create_api_token(user: User | None):
+    from archivebox.api.models import APIToken
+
    if user and user.is_superuser:
        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
        if api_tokens.exists():
@@ -24,116 +22,95 @@ def get_or_create_api_token(user):
        else:
            # does not exist, create a new one
            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
-        
+
+        if api_token is None:
+            return None
        assert api_token.is_valid(), f"API token is not valid {api_token}"

        return api_token
    return None


-def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
+def auth_using_token(token: str | None, request: HttpRequest | None = None) -> User | None:
    """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
-    from api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
-    
-    user = None
+    from archivebox.api.models import APIToken  # lazy import model to avoid loading it at urls.py import time

-    submitted_empty_form = str(token).strip() in ('string', '', 'None', 'null')
+    user: User | None = None
+
+    submitted_empty_form = str(token).strip() in ("string", "", "None", "null")
    if not submitted_empty_form:
        try:
-            token = APIToken.objects.get(token=token)
-            if token.is_valid():
-                user = token.created_by
-                request._api_token = token
+            api_token = APIToken.objects.get(token=token)
+            if api_token.is_valid() and isinstance(api_token.created_by, User):
+                user = api_token.created_by
+                if request is not None:
+                    setattr(request, "_api_token", api_token)
        except APIToken.DoesNotExist:
            pass

-    if not user:
-        # print('[❌] Failed to authenticate API user using API Key:', request)
-        return None
-    
-    return cast(AbstractBaseUser, user)
+    return user

-def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
+
+def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
    """Given a username and password, check if they are valid and return the corresponding user"""
-    user = None
-    
-    submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
+    user: User | None = None
+
+    submitted_empty_form = (username, password) in (("string", "string"), ("", ""), (None, None))
    if not submitted_empty_form:
-        user = authenticate(
+        authenticated_user = authenticate(
            username=username,
            password=password,
        )
-
-    if not user:
-        # print('[❌] Failed to authenticate API user using API Key:', request)
-        user = None
-
-    return cast(AbstractBaseUser | None, user)
+        if isinstance(authenticated_user, User):
+            user = authenticated_user
+    return user


 ### Base Auth Types


-class APITokenAuthCheck:
-    """The base class for authentication methods that use an api.models.APIToken"""
-    def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        request.user = auth_using_token(
-            token=key,
-            request=request,
-        )
-        if request.user and request.user.pk:
-            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
-            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
-            request._api_auth_method = self.__class__.__name__
-
-            if not request.user.is_superuser:
-                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
-        return request.user
-
-
-class UserPassAuthCheck:
-    """The base class for authentication methods that use a username & password"""
-    def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        request.user = auth_using_password(
-            username=username,
-            password=password,
-            request=request,
-        )
-        if request.user and request.user.pk:
-            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
-            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
-            request._api_auth_method = self.__class__.__name__
-
-            if not request.user.is_superuser:
-                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
-
-        return request.user
+def _require_superuser(user: User | None, request: HttpRequest, auth_method: str) -> User | None:
+    if user and user.pk:
+        request.user = user
+        setattr(request, "_api_auth_method", auth_method)
+        if not user.is_superuser:
+            raise HttpError(403, "Valid credentials but User does not have permission (make sure user.is_superuser=True)")
+    return user


 ### Django-Ninja-Provided Auth Methods

-class HeaderTokenAuth(APITokenAuthCheck, APIKeyHeader):
+
+class HeaderTokenAuth(APIKeyHeader):
    """Allow authenticating by passing X-API-Key=xyz as a request header"""
+
    param_name = "X-ArchiveBox-API-Key"

-class BearerTokenAuth(APITokenAuthCheck, HttpBearer):
-    """Allow authenticating by passing Bearer=xyz as a request header"""
-    pass
+    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
+        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)

-class QueryParamTokenAuth(APITokenAuthCheck, APIKeyQuery):
+
+class BearerTokenAuth(HttpBearer):
+    """Allow authenticating by passing Bearer=xyz as a request header"""
+
+    def authenticate(self, request: HttpRequest, token: str) -> User | None:
+        return _require_superuser(auth_using_token(token=token, request=request), request, self.__class__.__name__)
+
+
+class QueryParamTokenAuth(APIKeyQuery):
    """Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
+
    param_name = "api_key"

-class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
-    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
-    pass
+    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
+        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
+

 ### Enabled Auth Methods

 API_AUTH_METHODS = [
    HeaderTokenAuth(),
    BearerTokenAuth(),
-    QueryParamTokenAuth(), 
+    QueryParamTokenAuth(),
    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
-    UsernameAndPasswordAuth(),
 ]
@@ -0,0 +1,32 @@
+__package__ = "archivebox.api"
+
+from django.http import HttpResponse
+
+
+class ApiCorsMiddleware:
+    """Attach permissive CORS headers for API routes (token-based auth)."""
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        if request.path.startswith("/api/"):
+            if request.method == "OPTIONS" and request.META.get("HTTP_ACCESS_CONTROL_REQUEST_METHOD"):
+                response = HttpResponse(status=204)
+                return self._add_cors_headers(request, response)
+
+            response = self.get_response(request)
+            return self._add_cors_headers(request, response)
+
+        return self.get_response(request)
+
+    def _add_cors_headers(self, request, response):
+        origin = request.META.get("HTTP_ORIGIN")
+        if not origin:
+            return response
+
+        response["Access-Control-Allow-Origin"] = "*"
+        response["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
+        response["Access-Control-Allow-Headers"] = "Authorization, X-ArchiveBox-API-Key, Content-Type, X-CSRFToken"
+        response["Access-Control-Max-Age"] = "600"
+        return response
@@ -1,29 +1,239 @@
-# Generated by Django 4.2.11 on 2024-04-25 04:19
+# Generated by hand on 2025-12-29
+# Creates APIToken and OutboundWebhook tables using raw SQL

-import api.models
-from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
-import uuid
+import django.utils.timezone
+from django.conf import settings
+from archivebox.uuid_compat import uuid7
+from archivebox.base_models.models import get_or_create_system_user_pk
+import archivebox.api.models
+import signal_webhooks.fields
+import signal_webhooks.utils


 class Migration(migrations.Migration):
-
    initial = True

    dependencies = [
+        ("auth", "0012_alter_user_first_name_max_length"),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
-        migrations.CreateModel(
-            name='APIToken',
-            fields=[
-                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('token', models.CharField(default=api.models.generate_secret_token, max_length=32, unique=True)),
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('expires', models.DateTimeField(blank=True, null=True)),
-                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    sql="""
+                -- Create api_apitoken table
+                CREATE TABLE IF NOT EXISTS api_apitoken (
+                    id TEXT PRIMARY KEY NOT NULL,
+                    created_at DATETIME NOT NULL,
+                    modified_at DATETIME NOT NULL,
+
+                    token VARCHAR(32) NOT NULL UNIQUE,
+                    expires DATETIME,
+
+                    created_by_id INTEGER NOT NULL,
+
+                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
+                );
+                CREATE INDEX IF NOT EXISTS api_apitoken_created_by_id_idx ON api_apitoken(created_by_id);
+                CREATE INDEX IF NOT EXISTS api_apitoken_created_at_idx ON api_apitoken(created_at);
+                CREATE INDEX IF NOT EXISTS api_apitoken_token_idx ON api_apitoken(token);
+
+                -- Create api_outboundwebhook table
+                CREATE TABLE IF NOT EXISTS api_outboundwebhook (
+                    id TEXT PRIMARY KEY NOT NULL,
+                    created_at DATETIME NOT NULL,
+                    modified_at DATETIME NOT NULL,
+
+                    name VARCHAR(255) NOT NULL UNIQUE,
+                    signal VARCHAR(255) NOT NULL,
+                    ref VARCHAR(1024) NOT NULL,
+                    endpoint VARCHAR(2048) NOT NULL,
+                    headers TEXT NOT NULL DEFAULT '{}',
+                    auth_token TEXT NOT NULL DEFAULT '',
+                    enabled BOOLEAN NOT NULL DEFAULT 1,
+                    keep_last_response BOOLEAN NOT NULL DEFAULT 0,
+                    created DATETIME NOT NULL,
+                    updated DATETIME NOT NULL,
+                    last_response TEXT NOT NULL DEFAULT '',
+                    last_success DATETIME,
+                    last_failure DATETIME,
+
+                    created_by_id INTEGER NOT NULL,
+
+                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
+                );
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_by_id_idx ON api_outboundwebhook(created_by_id);
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
+                    """,
+                    reverse_sql="""
+                DROP TABLE IF EXISTS api_outboundwebhook;
+                DROP TABLE IF EXISTS api_apitoken;
+                    """,
+                ),
+            ],
+            state_operations=[
+                migrations.CreateModel(
+                    name="APIToken",
+                    fields=[
+                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
+                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+                        ("modified_at", models.DateTimeField(auto_now=True)),
+                        ("token", models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
+                        ("expires", models.DateTimeField(blank=True, null=True)),
+                        (
+                            "created_by",
+                            models.ForeignKey(
+                                default=get_or_create_system_user_pk,
+                                on_delete=django.db.models.deletion.CASCADE,
+                                to=settings.AUTH_USER_MODEL,
+                            ),
+                        ),
+                    ],
+                    options={
+                        "verbose_name": "API Key",
+                        "verbose_name_plural": "API Keys",
+                        "app_label": "api",
+                    },
+                ),
+                migrations.CreateModel(
+                    name="OutboundWebhook",
+                    fields=[
+                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
+                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+                        ("modified_at", models.DateTimeField(auto_now=True)),
+                        (
+                            "name",
+                            models.CharField(db_index=True, help_text="Webhook name.", max_length=255, unique=True, verbose_name="name"),
+                        ),
+                        (
+                            "signal",
+                            models.CharField(
+                                choices=[
+                                    ("CREATE", "Create"),
+                                    ("UPDATE", "Update"),
+                                    ("DELETE", "Delete"),
+                                    ("M2M", "M2M changed"),
+                                    ("CREATE_OR_UPDATE", "Create or Update"),
+                                    ("CREATE_OR_DELETE", "Create or Delete"),
+                                    ("CREATE_OR_M2M", "Create or M2M changed"),
+                                    ("UPDATE_OR_DELETE", "Update or Delete"),
+                                    ("UPDATE_OR_M2M", "Update or M2M changed"),
+                                    ("DELETE_OR_M2M", "Delete or M2M changed"),
+                                    ("CREATE_UPDATE_OR_DELETE", "Create, Update or Delete"),
+                                    ("CREATE_UPDATE_OR_M2M", "Create, Update or M2M changed"),
+                                    ("CREATE_DELETE_OR_M2M", "Create, Delete or M2M changed"),
+                                    ("UPDATE_DELETE_OR_M2M", "Update, Delete or M2M changed"),
+                                    ("CREATE_UPDATE_DELETE_OR_M2M", "Create, Update or Delete, or M2M changed"),
+                                ],
+                                help_text="Signal the webhook fires to.",
+                                max_length=255,
+                                verbose_name="signal",
+                            ),
+                        ),
+                        (
+                            "ref",
+                            models.CharField(
+                                db_index=True,
+                                help_text="Dot import notation to the model the webhook is for.",
+                                max_length=1023,
+                                validators=[signal_webhooks.utils.model_from_reference],
+                                verbose_name="referenced model",
+                            ),
+                        ),
+                        (
+                            "endpoint",
+                            models.URLField(help_text="Target endpoint for this webhook.", max_length=2047, verbose_name="endpoint"),
+                        ),
+                        (
+                            "headers",
+                            models.JSONField(
+                                blank=True,
+                                default=dict,
+                                help_text="Headers to send with the webhook request.",
+                                validators=[signal_webhooks.utils.is_dict],
+                                verbose_name="headers",
+                            ),
+                        ),
+                        (
+                            "auth_token",
+                            signal_webhooks.fields.TokenField(
+                                blank=True,
+                                default="",
+                                help_text="Authentication token to use in an Authorization header.",
+                                max_length=8000,
+                                validators=[signal_webhooks.utils.decode_cipher_key],
+                                verbose_name="authentication token",
+                            ),
+                        ),
+                        ("enabled", models.BooleanField(default=True, help_text="Is this webhook enabled?", verbose_name="enabled")),
+                        (
+                            "keep_last_response",
+                            models.BooleanField(
+                                default=False,
+                                help_text="Should the webhook keep a log of the latest response it got?",
+                                verbose_name="keep last response",
+                            ),
+                        ),
+                        (
+                            "created",
+                            models.DateTimeField(auto_now_add=True, help_text="When the webhook was created.", verbose_name="created"),
+                        ),
+                        (
+                            "updated",
+                            models.DateTimeField(auto_now=True, help_text="When the webhook was last updated.", verbose_name="updated"),
+                        ),
+                        (
+                            "last_response",
+                            models.CharField(
+                                blank=True,
+                                default="",
+                                help_text="Latest response to this webhook.",
+                                max_length=8000,
+                                verbose_name="last response",
+                            ),
+                        ),
+                        (
+                            "last_success",
+                            models.DateTimeField(
+                                default=None,
+                                help_text="When the webhook last succeeded.",
+                                null=True,
+                                verbose_name="last success",
+                            ),
+                        ),
+                        (
+                            "last_failure",
+                            models.DateTimeField(
+                                default=None,
+                                help_text="When the webhook last failed.",
+                                null=True,
+                                verbose_name="last failure",
+                            ),
+                        ),
+                        (
+                            "created_by",
+                            models.ForeignKey(
+                                default=get_or_create_system_user_pk,
+                                on_delete=django.db.models.deletion.CASCADE,
+                                to=settings.AUTH_USER_MODEL,
+                            ),
+                        ),
+                    ],
+                    options={
+                        "verbose_name": "API Outbound Webhook",
+                        "app_label": "api",
+                    },
+                ),
+                migrations.AddConstraint(
+                    model_name="outboundwebhook",
+                    constraint=models.UniqueConstraint(fields=["ref", "endpoint"], name="prevent_duplicate_hooks_api_outboundwebhook"),
+                ),
            ],
        ),
    ]
@@ -1,17 +0,0 @@
-# Generated by Django 5.0.4 on 2024-04-26 05:28
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name='apitoken',
-            options={'verbose_name': 'API Key', 'verbose_name_plural': 'API Keys'},
-        ),
-    ]
@@ -1,77 +0,0 @@
-# Generated by Django 5.0.6 on 2024-06-03 01:52
-
-import abid_utils.models
-import charidfield.fields
-import django.db.models.deletion
-import signal_webhooks.fields
-import signal_webhooks.utils
-import uuid
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0002_alter_apitoken_options'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='user',
-            new_name='created_by',
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='abid',
-            field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='apt_', unique=True),
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='modified',
-            field=models.DateTimeField(auto_now=True),
-        ),
-        migrations.AddField(
-            model_name='apitoken',
-            name='uuid',
-            field=models.UUIDField(blank=True, null=True, unique=True),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False),
-        ),
-        migrations.CreateModel(
-            name='OutboundWebhook',
-            fields=[
-                ('name', models.CharField(db_index=True, help_text='Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).', max_length=255, unique=True, verbose_name='name')),
-                ('signal', models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='The type of event the webhook should fire for (e.g. Create, Update, Delete).', max_length=255, verbose_name='signal')),
-                ('ref', models.CharField(db_index=True, help_text='Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model')),
-                ('endpoint', models.URLField(help_text='External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).', max_length=2047, verbose_name='endpoint')),
-                ('headers', models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers')),
-                ('auth_token', signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token')),
-                ('enabled', models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled')),
-                ('keep_last_response', models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response')),
-                ('updated', models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated')),
-                ('last_response', models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response')),
-                ('last_success', models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success')),
-                ('last_failure', models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure')),
-                ('created', models.DateTimeField(auto_now_add=True)),
-                ('modified', models.DateTimeField(auto_now=True)),
-                ('id', models.UUIDField(blank=True, null=True, unique=True)),
-                ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
-                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='whk_', unique=True)),
-                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-            options={
-                'verbose_name': 'API Outbound Webhook',
-                'abstract': False,
-            },
-        ),
-        migrations.AddConstraint(
-            model_name='outboundwebhook',
-            constraint=models.UniqueConstraint(fields=('ref', 'endpoint'), name='prevent_duplicate_hooks_api_outboundwebhook'),
-        ),
-    ]
@@ -1,24 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 10:44
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0003_rename_user_apitoken_created_by_apitoken_abid_and_more'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='uuid',
-            field=models.UUIDField(blank=True, editable=False, null=True, unique=True),
-        ),
-    ]
@@ -1,22 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:40
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0004_alter_apitoken_id_alter_apitoken_uuid'),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name='apitoken',
-            name='uuid',
-        ),
-        migrations.RemoveField(
-            model_name='outboundwebhook',
-            name='id',
-        ),
-    ]
@@ -1,29 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:43
-
-import uuid
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more'),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='outboundwebhook',
-            old_name='uuid',
-            new_name='id'
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
-        ),
-    ]
@@ -1,22 +0,0 @@
-# Generated by Django 5.1 on 2024-08-20 22:52
-
-import abid_utils.models
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0006_remove_outboundwebhook_uuid_apitoken_id_and_more'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created_by',
-            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-    ]
@@ -1,47 +0,0 @@
-# Generated by Django 5.1 on 2024-09-04 23:32
-
-import abid_utils.models
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0007_alter_apitoken_created_by'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created',
-            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created_by',
-            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created',
-            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created_by',
-            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='id',
-            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
-        ),
-    ]
@@ -1,39 +0,0 @@
-# Generated by Django 5.1 on 2024-09-05 00:26
-
-import abid_utils.models
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0008_alter_apitoken_created_alter_apitoken_created_by_and_more'),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='created',
-            new_name='created_at',
-        ),
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='modified',
-            new_name='modified_at',
-        ),
-        migrations.RenameField(
-            model_name='outboundwebhook',
-            old_name='modified',
-            new_name='modified_at',
-        ),
-        migrations.AddField(
-            model_name='outboundwebhook',
-            name='created_at',
-            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created',
-            field=models.DateTimeField(auto_now_add=True, help_text='When the webhook was created.', verbose_name='created'),
-        ),
-    ]
@@ -1,127 +1,54 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

 import secrets
-from datetime import timedelta
+from archivebox.uuid_compat import uuid7

 from django.conf import settings
 from django.db import models
 from django.utils import timezone
-
+from django_stubs_ext.db.models import TypedModelMeta
 from signal_webhooks.models import WebhookBase

-from django_stubs_ext.db.models import TypedModelMeta
-
-from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
-
+from archivebox.base_models.models import get_or_create_system_user_pk


 def generate_secret_token() -> str:
-    # returns cryptographically secure string with len() == 32
    return secrets.token_hex(16)


-class APIToken(ABIDModel):
-    """
-    A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox.
-    """
-    # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
-    abid_prefix = 'apt_'
-    abid_ts_src = 'self.created_at'
-    abid_uri_src = 'self.created_by_id'
-    abid_subtype_src = '"01"'
-    abid_rand_src = 'self.id'
-    abid_drift_allowed = True
-
-    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
-    abid = ABIDField(prefix=abid_prefix)
-
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+class APIToken(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)
-
    token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
    expires = models.DateTimeField(null=True, blank=True)

    class Meta(TypedModelMeta):
+        app_label = "api"
        verbose_name = "API Key"
        verbose_name_plural = "API Keys"

    def __str__(self) -> str:
        return self.token

-    def __repr__(self) -> str:
-        return f'<APIToken user={self.created_by.username} token={self.token_redacted}>'
-
-    def __json__(self) -> dict:
-        return {
-            "TYPE":             "APIToken",    
-            "id":               str(self.pk),
-            "abid":             str(self.ABID),
-            "created_by_id":    str(self.created_by_id),
-            "token":            self.token,
-            "created_at":       self.created_at.isoformat(),
-            "expires":          self.expires_as_iso8601,
-        }
-
-    @property
-    def expires_as_iso8601(self):
-        """Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
-        expiry_date = self.expires or (timezone.now() + timedelta(days=365 * 100))
-
-        return expiry_date.isoformat()
-    
    @property
    def token_redacted(self):
-        return f'************{self.token[-4:]}'
+        return f"************{self.token[-4:]}"

    def is_valid(self, for_date=None):
-        for_date = for_date or timezone.now()
-
-        if self.expires and self.expires < for_date:
-            return False
-
-        return True
+        return not self.expires or self.expires >= (for_date or timezone.now())


-
-
-
-
-# monkey patch django-signals-webhooks to change how it shows up in Admin UI
-
-class OutboundWebhook(ABIDModel, WebhookBase):
-    """
-    Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using:
-        settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
-    """
-    abid_prefix = 'whk_'
-    abid_ts_src = 'self.created_at'
-    abid_uri_src = 'self.endpoint'
-    abid_subtype_src = 'self.ref'
-    abid_rand_src = 'self.id'
-    abid_drift_allowed = True
-
-    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
-    abid = ABIDField(prefix=abid_prefix)
-
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
+class OutboundWebhook(WebhookBase):
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)

-    # More fields here: WebhookBase...
-
-    WebhookBase._meta.get_field('name').help_text = (
-        'Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).')
-    WebhookBase._meta.get_field('signal').help_text = (
-        'The type of event the webhook should fire for (e.g. Create, Update, Delete).')
-    WebhookBase._meta.get_field('ref').help_text = (
-        'Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).')
-    WebhookBase._meta.get_field('endpoint').help_text = (
-        'External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).')
-
    class Meta(WebhookBase.Meta):
-        verbose_name = 'API Outbound Webhook'
-
+        app_label = "api"
+        verbose_name = "API Outbound Webhook"

    def __str__(self) -> str:
-        return f'[{self.abid}] {self.ref} -> {self.endpoint}'
+        return f"[{self.id}] {self.ref} -> {self.endpoint}"
@@ -1,30 +0,0 @@
-__package__ = 'archivebox.api'
-
-# from django.test import TestCase
-# from ninja.testing import TestClient
-
-# from .routes_cli import router
-
-# class ArchiveBoxCLIAPITestCase(TestCase):
-#     def setUp(self):
-#         self.client = TestClient(router)
-
-#     def test_add_endpoint(self):
-#         response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
-
-#     def test_remove_endpoint(self):
-#         response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
-
-#     def test_update_endpoint(self):
-#         response = self.client.post("/update", json={})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
-
-#     def test_list_all_endpoint(self):
-#         response = self.client.post("/list_all", json={})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
@@ -1,16 +1,28 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

+from urllib.parse import quote
+
+from django.http import HttpRequest, HttpResponseRedirect
+from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView

+from archivebox.core.host_utils import build_web_url
 from .v1_api import urls as v1_api_urls

+
+def archive_redirect_view(request: HttpRequest, url: str) -> HttpResponseRedirect:
+    if request.META.get("QUERY_STRING"):
+        url = f"{url}?{request.META['QUERY_STRING']}"
+    return redirect(build_web_url(f"/web/{quote(url, safe=':/')}", request=request))
+
+
 urlpatterns = [
-    path("",                 RedirectView.as_view(url='/api/v1')),
-
-    path("v1/",              v1_api_urls),
-    path("v1",               RedirectView.as_view(url='/api/v1/docs')),
-
+    path("", RedirectView.as_view(url="/api/v1/docs")),
+    path("archive/<path:url>", archive_redirect_view, name="api-archive-redirect"),
+    path("v1/", RedirectView.as_view(url="/api/v1/docs")),
+    path("v1/", v1_api_urls),
+    path("v1", RedirectView.as_view(url="/api/v1/docs")),
    # ... v2 can be added here ...
    # path("v2/",              v2_api_urls),
    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
@@ -1,4 +1,4 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"


 from io import StringIO
@@ -7,6 +7,7 @@ from contextlib import redirect_stdout, redirect_stderr

 from django.http import HttpRequest, HttpResponse
 from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
+from django.contrib.auth.models import User

 from ninja import NinjaAPI, Swagger

@@ -15,12 +16,13 @@ from ninja import NinjaAPI, Swagger
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH

-from api.auth import API_AUTH_METHODS
+from archivebox.api.auth import API_AUTH_METHODS
+from archivebox.api.models import APIToken


-COMMIT_HASH = get_COMMIT_HASH() or 'unknown'
+COMMIT_HASH = get_COMMIT_HASH() or "unknown"

-html_description=f'''
+html_description = f"""
 <h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
 <br/>
 <i><b>WARNING: This API is still in an early development stage and may change!</b></i>
@@ -33,44 +35,48 @@ html_description=f'''
 <li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
 </ul>
 <small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
-'''
+"""


 def register_urls(api: NinjaAPI) -> NinjaAPI:
-    api.add_router('/auth/',     'api.v1_auth.router')
-    api.add_router('/core/',     'api.v1_core.router')
-    api.add_router('/cli/',      'api.v1_cli.router')
+    api.add_router("/auth/", "archivebox.api.v1_auth.router")
+    api.add_router("/core/", "archivebox.api.v1_core.router")
+    api.add_router("/crawls/", "archivebox.api.v1_crawls.router")
+    api.add_router("/cli/", "archivebox.api.v1_cli.router")
+    api.add_router("/machine/", "archivebox.api.v1_machine.router")
+    api.add_router("/personas/", "archivebox.api.v1_personas.router")
    return api


-class NinjaAPIWithIOCapture(NinjaAPI):    
+class NinjaAPIWithIOCapture(NinjaAPI):
    def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
        stdout, stderr = StringIO(), StringIO()

        with redirect_stderr(stderr):
            with redirect_stdout(stdout):
-                request.stdout = stdout
-                request.stderr = stderr
+                setattr(request, "stdout", stdout)
+                setattr(request, "stderr", stderr)

                response = super().create_temporal_response(request)

-        # Diable caching of API responses entirely
-        response['Cache-Control'] = 'no-store'
+        # Disable caching of API responses entirely
+        response["Cache-Control"] = "no-store"

        # Add debug stdout and stderr headers to response
-        response['X-ArchiveBox-Stdout'] = str(request.stdout)[200:]
-        response['X-ArchiveBox-Stderr'] = str(request.stderr)[200:]
+        response["X-ArchiveBox-Stdout"] = stdout.getvalue().replace("\n", "\\n")[:200]
+        response["X-ArchiveBox-Stderr"] = stderr.getvalue().replace("\n", "\\n")[:200]
        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'

        # Add Auth Headers to response
-        api_token = getattr(request, '_api_token', None)
-        token_expiry = api_token.expires.isoformat() if api_token else 'Never'
+        api_token_attr = getattr(request, "_api_token", None)
+        api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
+        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"

-        response['X-ArchiveBox-Auth-Method'] = getattr(request, '_api_auth_method', None) or 'None'
-        response['X-ArchiveBox-Auth-Expires'] = token_expiry
-        response['X-ArchiveBox-Auth-Token-Id'] = api_token.abid if api_token else 'None'
-        response['X-ArchiveBox-Auth-User-Id'] = request.user.pk if request.user.pk else 'None'
-        response['X-ArchiveBox-Auth-User-Username'] = request.user.username if request.user.pk else 'None'
+        response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
+        response["X-ArchiveBox-Auth-Expires"] = token_expiry
+        response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
+        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
+        response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"

        # import ipdb; ipdb.set_trace()
        # print('RESPONDING NOW', response)
@@ -79,10 +85,9 @@ class NinjaAPIWithIOCapture(NinjaAPI):


 api = NinjaAPIWithIOCapture(
-    title='ArchiveBox API',
+    title="ArchiveBox API",
    description=html_description,
-    version='1.0.0',
-    csrf=False,
+    version=VERSION,
    auth=API_AUTH_METHODS,
    urls_namespace="api-1",
    docs=Swagger(settings={"persistAuthorization": True}),
@@ -99,15 +104,15 @@ def generic_exception_handler(request, err):
    if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
        status = 404

-    print(''.join(format_exception(err)))
+    print("".join(format_exception(err)))

    return api.create_response(
        request,
        {
            "succeeded": False,
-            "message": f'{err.__class__.__name__}: {err}',
+            "message": f"{err.__class__.__name__}: {err}",
            "errors": [
-                ''.join(format_exception(err)),
+                "".join(format_exception(err)),
                # or send simpler parent-only traceback:
                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
            ],
@@ -116,7 +121,6 @@ def generic_exception_handler(request, err):
    )


-
 # import orjson
 # from ninja.renderers import BaseRenderer
 # class ORJSONRenderer(BaseRenderer):
@@ -1,26 +1,28 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

-from typing import Optional
+from django.http import HttpRequest

 from ninja import Router, Schema
-from django.utils import timezone
-from datetime import timedelta

-from api.models import APIToken
-from api.auth import auth_using_token, auth_using_password, get_or_create_api_token
+from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token


-router = Router(tags=['Authentication'], auth=None)
+router = Router(tags=["Authentication"], auth=None)


 class PasswordAuthSchema(Schema):
    """Schema for a /get_api_token request"""
-    username: Optional[str] = None
-    password: Optional[str] = None
+
+    username: str | None = None
+    password: str | None = None


-@router.post("/get_api_token", auth=None, summary='Generate an API token for a given username & password (or currently logged-in user)')             # auth=None because they are not authed yet
-def get_api_token(request, auth_data: PasswordAuthSchema):
+@router.post(
+    "/get_api_token",
+    auth=None,
+    summary="Generate an API token for a given username & password (or currently logged-in user)",
+)  # auth=None because they are not authed yet
+def get_api_token(request: HttpRequest, auth_data: PasswordAuthSchema):
    user = auth_using_password(
        username=auth_data.username,
        password=auth_data.password,
@@ -30,24 +32,34 @@ def get_api_token(request, auth_data: PasswordAuthSchema):
    if user and user.is_superuser:
        api_token = get_or_create_api_token(user)
        assert api_token is not None, "Failed to create API token"
-        return api_token.__json__()
-    
-    return {"success": False, "errors": ["Invalid credentials"]}
+        return {
+            "success": True,
+            "user_id": str(user.pk),
+            "username": user.username,
+            "token": api_token.token,
+            "expires": api_token.expires.isoformat() if api_token.expires else None,
+        }

+    return {"success": False, "errors": ["Invalid credentials"]}


 class TokenAuthSchema(Schema):
    """Schema for a /check_api_token request"""
+
    token: str


-@router.post("/check_api_token", auth=None, summary='Validate an API token to make sure its valid and non-expired')        # auth=None because they are not authed yet
-def check_api_token(request, token_data: TokenAuthSchema):
+@router.post(
+    "/check_api_token",
+    auth=None,
+    summary="Validate an API token to make sure its valid and non-expired",
+)  # auth=None because they are not authed yet
+def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
    user = auth_using_token(
        token=token_data.token,
        request=request,
    )
    if user:
        return {"success": True, "user_id": str(user.pk)}
-    
+
    return {"success": False, "user_id": None}
@@ -1,169 +1,200 @@
-__package__ = 'archivebox.api'
+__package__ = "archivebox.api"

-from typing import List, Dict, Any, Optional
+import json
+from io import StringIO
+from typing import Any
 from enum import Enum

+from django.http import HttpRequest
+
 from ninja import Router, Schema
+from pydantic import Field

-from ..main import (
-    add,
-    remove,
-    update,
-    list_all,
-    schedule,
-)
 from archivebox.misc.util import ansi_to_html
-from archivebox.config.common import ARCHIVING_CONFIG
+from archivebox.config.common import get_config


-from .auth import API_AUTH_METHODS
+# from .auth import API_AUTH_METHODS

 # router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=['ArchiveBox CLI Sub-Commands'], auth=API_AUTH_METHODS)
+router = Router(tags=["ArchiveBox CLI Sub-Commands"])


 # Schemas

-JSONType = List[Any] | Dict[str, Any] | bool | int | str | None
+JSONType = list[Any] | dict[str, Any] | bool | int | str | None
+

 class CLICommandResponseSchema(Schema):
    success: bool
-    errors: List[str]
+    errors: list[str]
    result: JSONType
+    result_format: str = "str"
    stdout: str
    stderr: str

+
 class FilterTypeChoices(str, Enum):
-    exact = 'exact'
-    substring = 'substring'
-    regex = 'regex'
-    domain = 'domain'
-    tag = 'tag'
-    timestamp = 'timestamp'
+    exact = "exact"
+    substring = "substring"
+    regex = "regex"
+    domain = "domain"
+    tag = "tag"
+    timestamp = "timestamp"
+

 class StatusChoices(str, Enum):
-    indexed = 'indexed'
-    archived = 'archived'
-    unarchived = 'unarchived'
-    present = 'present'
-    valid = 'valid'
-    invalid = 'invalid'
-    duplicate = 'duplicate'
-    orphaned = 'orphaned'
-    corrupted = 'corrupted'
-    unrecognized = 'unrecognized'
+    indexed = "indexed"
+    archived = "archived"
+    unarchived = "unarchived"
+    present = "present"
+    valid = "valid"
+    invalid = "invalid"
+    duplicate = "duplicate"
+    orphaned = "orphaned"
+    corrupted = "corrupted"
+    unrecognized = "unrecognized"


 class AddCommandSchema(Schema):
-    urls: List[str]
+    urls: list[str]
+    snapshot_ids: list[str] | None = None
    tag: str = ""
    depth: int = 0
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW  # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW
-    update_all: bool = False
-    index_only: bool = False
-    overwrite: bool = False
-    init: bool = False
-    extractors: str = ""
+    max_urls: int = 0
+    crawl_max_size: int = 0
+    crawl_timeout: int = 0
+    snapshot_max_size: int = 0
    parser: str = "auto"
+    plugins: str = ""
+    update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
+    overwrite: bool = False
+    index_only: bool = False
+

 class UpdateCommandSchema(Schema):
-    resume: Optional[float] = 0
-    only_new: bool = ARCHIVING_CONFIG.ONLY_NEW
-    index_only: bool = False
-    overwrite: bool = False
-    after: Optional[float] = 0
-    before: Optional[float] = 999999999999999
-    status: Optional[StatusChoices] = StatusChoices.unarchived
-    filter_type: Optional[str] = FilterTypeChoices.substring
-    filter_patterns: Optional[List[str]] = ['https://example.com']
-    extractors: Optional[str] = ""
+    resume: str | None = None
+    after: float | None = 0
+    before: float | None = 999999999999999
+    filter_type: str | None = FilterTypeChoices.substring
+    filter_patterns: list[str] | None = ["https://example.com"]
+    batch_size: int = 100
+    continuous: bool = False
+

 class ScheduleCommandSchema(Schema):
-    import_path: Optional[str] = None
+    import_path: str | None = None
    add: bool = False
-    every: Optional[str] = None
-    tag: str = ''
+    show: bool = False
+    foreground: bool = False
+    run_all: bool = False
+    quiet: bool = False
+    every: str | None = None
+    tag: str = ""
    depth: int = 0
    overwrite: bool = False
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW
+    update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
    clear: bool = False

+
 class ListCommandSchema(Schema):
-    filter_patterns: Optional[List[str]] = ['https://example.com']
+    filter_patterns: list[str] | None = ["https://example.com"]
    filter_type: str = FilterTypeChoices.substring
-    status: Optional[StatusChoices] = StatusChoices.indexed
-    after: Optional[float] = 0
-    before: Optional[float] = 999999999999999
-    sort: str = 'bookmarked_at'
+    status: StatusChoices = StatusChoices.indexed
+    after: float | None = 0
+    before: float | None = 999999999999999
+    sort: str = "bookmarked_at"
    as_json: bool = True
    as_html: bool = False
-    as_csv: str | bool = 'timestamp,url'
+    as_csv: str | None = "timestamp,url"
    with_headers: bool = False

+
 class RemoveCommandSchema(Schema):
-    delete: bool = True
-    after: Optional[float] = 0
-    before: Optional[float] = 999999999999999
+    after: float | None = 0
+    before: float | None = 999999999999999
    filter_type: str = FilterTypeChoices.exact
-    filter_patterns: Optional[List[str]] = ['https://example.com']
+    filter_patterns: list[str] | None = ["https://example.com"]


+@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
+def cli_add(request: HttpRequest, args: AddCommandSchema):
+    from archivebox.cli.archivebox_add import add

-
-
-@router.post("/add", response=CLICommandResponseSchema, summary='archivebox add [args] [urls]')
-def cli_add(request, args: AddCommandSchema):
-    result = add(
+    crawl, snapshots = add(
        urls=args.urls,
+        snapshot_ids=args.snapshot_ids,
        tag=args.tag,
        depth=args.depth,
+        max_urls=args.max_urls,
+        crawl_max_size=args.crawl_max_size,
+        crawl_timeout=args.crawl_timeout,
+        snapshot_max_size=args.snapshot_max_size,
        update=args.update,
-        update_all=args.update_all,
        index_only=args.index_only,
        overwrite=args.overwrite,
-        init=args.init,
-        extractors=args.extractors,
+        plugins=args.plugins,
        parser=args.parser,
+        bg=True,  # Always run in background for API calls
+        created_by_id=request.user.pk,
    )

+    snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
+    result_payload = {
+        "crawl_id": str(crawl.id),
+        "num_snapshots": len(snapshot_ids),
+        "snapshot_ids": snapshot_ids,
+        "queued_urls": args.urls,
+    }
+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
+
    return {
        "success": True,
        "errors": [],
-        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+        "result": result_payload,
+        "result_format": "json",
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }


-@router.post("/update", response=CLICommandResponseSchema, summary='archivebox update [args] [filter_patterns]')
-def cli_update(request, args: UpdateCommandSchema):
+@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
+def cli_update(request: HttpRequest, args: UpdateCommandSchema):
+    from archivebox.cli.archivebox_update import update
+
    result = update(
-        resume=args.resume,
-        only_new=args.only_new,
-        index_only=args.index_only,
-        overwrite=args.overwrite,
-        before=args.before,
+        filter_patterns=args.filter_patterns or [],
+        filter_type=args.filter_type or FilterTypeChoices.substring,
        after=args.after,
-        status=args.status,
-        filter_type=args.filter_type,
-        filter_patterns=args.filter_patterns,
-        extractors=args.extractors,
+        before=args.before,
+        resume=args.resume,
+        batch_size=args.batch_size,
+        continuous=args.continuous,
+        stop_daemon_stack=False,
    )
+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
    return {
        "success": True,
        "errors": [],
        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }


-@router.post("/schedule", response=CLICommandResponseSchema, summary='archivebox schedule [args] [import_path]')
-def cli_schedule(request, args: ScheduleCommandSchema):
+@router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
+def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
+    from archivebox.cli.archivebox_schedule import schedule
+
    result = schedule(
        import_path=args.import_path,
        add=args.add,
        show=args.show,
+        foreground=args.foreground,
+        run_all=args.run_all,
+        quiet=args.quiet,
        clear=args.clear,
        every=args.every,
        tag=args.tag,
@@ -172,19 +203,23 @@ def cli_schedule(request, args: ScheduleCommandSchema):
        update=args.update,
    )

+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
    return {
        "success": True,
        "errors": [],
        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+        "result_format": "json",
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }


+@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
+def cli_search(request: HttpRequest, args: ListCommandSchema):
+    from archivebox.cli.archivebox_search import search

-@router.post("/list", response=CLICommandResponseSchema, summary='archivebox list [args] [filter_patterns]')
-def cli_list(request, args: ListCommandSchema):
-    result = list_all(
+    result = search(
        filter_patterns=args.filter_patterns,
        filter_type=args.filter_type,
        status=args.status,
@@ -197,40 +232,63 @@ def cli_list(request, args: ListCommandSchema):
        with_headers=args.with_headers,
    )

-    result_format = 'txt'
+    result_format = "txt"
    if args.as_json:
        result_format = "json"
+        result = json.loads(result)
    elif args.as_html:
        result_format = "html"
    elif args.as_csv:
        result_format = "csv"

+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
    return {
        "success": True,
        "errors": [],
        "result": result,
        "result_format": result_format,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }
-    


-@router.post("/remove", response=CLICommandResponseSchema, summary='archivebox remove [args] [filter_patterns]')
-def cli_remove(request, args: RemoveCommandSchema):
-    result = remove(
-        yes=True,            # no way to interactively ask for confirmation via API, so we force yes
-        delete=args.delete,
+@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
+def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
+    from archivebox.cli.archivebox_remove import remove
+    from archivebox.cli.archivebox_search import get_snapshots
+    from archivebox.core.models import Snapshot
+
+    filter_patterns = args.filter_patterns or []
+    snapshots_to_remove = get_snapshots(
+        filter_patterns=filter_patterns,
+        filter_type=args.filter_type,
+        after=args.after,
+        before=args.before,
+    )
+    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
+
+    remove(
+        yes=True,  # no way to interactively ask for confirmation via API, so we force yes
+        snapshots=snapshots_to_remove,
        before=args.before,
        after=args.after,
        filter_type=args.filter_type,
-        filter_patterns=args.filter_patterns,
+        filter_patterns=filter_patterns,
    )
+
+    result = {
+        "removed_count": len(removed_snapshot_ids),
+        "removed_snapshot_ids": removed_snapshot_ids,
+        "remaining_snapshots": Snapshot.objects.count(),
+    }
+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
    return {
        "success": True,
        "errors": [],
        "result": result,
-        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
-        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+        "result_format": "json",
+        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
+        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
    }
-    
@@ -0,0 +1,239 @@
+__package__ = "archivebox.api"
+
+from pathlib import Path
+from uuid import UUID
+from datetime import datetime
+from django.http import FileResponse, HttpRequest
+from django.shortcuts import redirect
+from django.utils import timezone
+
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import User
+
+from ninja import Router, Schema
+from ninja.errors import HttpError
+
+from archivebox.core.models import Snapshot
+from archivebox.config.common import get_config
+from archivebox.crawls.models import Crawl
+
+from .auth import API_AUTH_METHODS, auth_using_token
+
+router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
+
+
+class CrawlSchema(Schema):
+    TYPE: str = "crawls.models.Crawl"
+
+    id: UUID
+
+    modified_at: datetime
+    created_at: datetime
+    created_by_id: str
+    created_by_username: str
+
+    status: str
+    retry_at: datetime | None
+    is_paused: bool
+
+    urls: str
+    max_depth: int
+    tags_str: str
+    config: dict
+
+    # snapshots: List[SnapshotSchema]
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+
+    @staticmethod
+    def resolve_created_by_username(obj):
+        user_model = get_user_model()
+        user = user_model.objects.get(id=obj.created_by_id)
+        username = getattr(user, "username", None)
+        return username if isinstance(username, str) else str(user)
+
+    @staticmethod
+    def resolve_snapshots(obj, context):
+        if bool(getattr(context["request"], "with_snapshots", False)):
+            return obj.snapshot_set.all().distinct()
+        return Snapshot.objects.none()
+
+
+class CrawlUpdateSchema(Schema):
+    action: str | None = None
+    status: str | None = None
+    retry_at: datetime | None = None
+    tags: list[str] | None = None
+    tags_str: str | None = None
+
+
+class CrawlCreateSchema(Schema):
+    urls: list[str]
+    max_depth: int = 0
+    tags: list[str] | None = None
+    tags_str: str = ""
+    label: str = ""
+    notes: str = ""
+    config: dict = {}
+
+
+class CrawlDeleteResponseSchema(Schema):
+    success: bool
+    crawl_id: str
+    deleted_count: int
+    deleted_snapshots: int
+
+
+def normalize_tag_list(tags: list[str] | None = None, tags_str: str = "") -> list[str]:
+    if tags is not None:
+        return [tag.strip() for tag in tags if tag and tag.strip()]
+    return [tag.strip() for tag in tags_str.split(",") if tag.strip()]
+
+
+@router.get("/crawls", response=list[CrawlSchema], url_name="get_crawls")
+def get_crawls(request: HttpRequest):
+    return Crawl.objects.all().distinct()
+
+
+@router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
+def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
+    urls = [url.strip() for url in data.urls if url and url.strip()]
+    if not urls:
+        raise HttpError(400, "At least one URL is required")
+    if data.max_depth not in (0, 1, 2, 3, 4):
+        raise HttpError(400, "max_depth must be between 0 and 4")
+
+    tags = normalize_tag_list(data.tags, data.tags_str)
+    config = dict(data.config or {})
+    config.setdefault("PERMISSIONS", str(get_config(user=request.user).PERMISSIONS))
+    crawl = Crawl.objects.create(
+        urls="\n".join(urls),
+        max_depth=data.max_depth,
+        tags_str=",".join(tags),
+        label=data.label,
+        notes=data.notes,
+        config=config,
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=timezone.now(),
+        created_by=request.user if isinstance(request.user, User) else None,
+    )
+    return crawl
+
+
+@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
+def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
+    """Get a specific Crawl by id."""
+    setattr(request, "with_snapshots", with_snapshots)
+    setattr(request, "with_archiveresults", with_archiveresults)
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
+
+    if crawl and as_rss:
+        query = request.GET.copy()
+        query.pop("as_rss", None)
+        query["crawl_id"] = str(crawl.id)
+        return redirect(f"/api/v1/core/snapshots.rss?{query.urlencode()}")
+
+    return crawl
+
+
+def crawl_file(request: HttpRequest, crawl_id: str, path: str):
+    user = getattr(request, "user", None)
+    is_superuser = bool(
+        getattr(user, "is_authenticated", False) and getattr(user, "is_active", False) and getattr(user, "is_superuser", False),
+    )
+    if not is_superuser:
+        token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
+        auth_header = request.headers.get("Authorization", "")
+        if not token and auth_header.lower().startswith("bearer "):
+            token = auth_header.split(None, 1)[1].strip()
+        token_user = auth_using_token(token=token, request=request) if token else None
+        is_superuser = bool(token_user and token_user.is_active and token_user.is_superuser)
+    if not is_superuser:
+        raise HttpError(403, "Permission denied")
+
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl_root = Path(crawl.output_dir).resolve()
+    file_path = (crawl_root / path).resolve()
+    if not file_path.is_file() or crawl_root not in file_path.parents:
+        raise HttpError(404, "Crawl file not found")
+
+    response = FileResponse(file_path.open("rb"))
+    response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
+    response["Pragma"] = "no-cache"
+    response["Expires"] = "0"
+    response["X-Content-Type-Options"] = "nosniff"
+    return response
+
+
+@router.get("/crawl/{crawl_id}/files/{filename}", auth=None, url_name="crawl_file_root")
+def crawl_file_root(request: HttpRequest, crawl_id: str, filename: str):
+    return crawl_file(request, crawl_id, filename)
+
+
+@router.get("/crawl/{crawl_id}/files/{folder}/{filename}", auth=None, url_name="crawl_file_nested_1")
+def crawl_file_nested_1(request: HttpRequest, crawl_id: str, folder: str, filename: str):
+    return crawl_file(request, crawl_id, f"{folder}/{filename}")
+
+
+@router.get("/crawl/{crawl_id}/files/{folder}/{subfolder}/{filename}", auth=None, url_name="crawl_file_nested_2")
+def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfolder: str, filename: str):
+    return crawl_file(request, crawl_id, f"{folder}/{subfolder}/{filename}")
+
+
+@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
+def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
+    """Update a crawl (e.g., set status=sealed to cancel queued work)."""
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    payload = data.dict(exclude_unset=True)
+    update_fields = ["modified_at"]
+
+    action = payload.pop("action", None)
+    if action:
+        if action == "pause":
+            crawl.pause()
+            return crawl
+        if action in ("resume", "unpause"):
+            crawl.resume()
+            return crawl
+        if action == "cancel":
+            crawl.cancel()
+            return crawl
+        raise HttpError(400, f"Invalid action: {action}")
+
+    tags = payload.pop("tags", None)
+    tags_str = payload.pop("tags_str", None)
+    if tags is not None or tags_str is not None:
+        crawl.tags_str = ",".join(normalize_tag_list(tags, tags_str or ""))
+        update_fields.append("tags_str")
+
+    if "status" in payload:
+        if payload["status"] not in Crawl.StatusChoices.values:
+            raise HttpError(400, f"Invalid status: {payload['status']}")
+        if payload["status"] == Crawl.StatusChoices.SEALED:
+            crawl.cancel()
+            return crawl
+        crawl.status = payload["status"]
+        update_fields.append("status")
+
+    if "retry_at" in payload:
+        crawl.retry_at = payload["retry_at"]
+        update_fields.append("retry_at")
+
+    crawl.save(update_fields=update_fields)
+    return crawl
+
+
+@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
+def delete_crawl(request: HttpRequest, crawl_id: str):
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    crawl_id_str = str(crawl.id)
+    snapshot_count = crawl.snapshot_set.count()
+    deleted_count, _ = crawl.delete()
+    return {
+        "success": True,
+        "crawl_id": crawl_id_str,
+        "deleted_count": deleted_count,
+        "deleted_snapshots": snapshot_count,
+    }
@@ -0,0 +1,161 @@
+__package__ = "archivebox.api"
+
+from uuid import UUID
+from typing import Annotated
+from datetime import datetime
+
+from django.http import HttpRequest
+
+from ninja import FilterLookup, FilterSchema, Query, Router, Schema
+from ninja.pagination import paginate
+
+from archivebox.api.v1_core import CustomPagination
+
+
+router = Router(tags=["Machine and Dependencies"])
+
+
+# ============================================================================
+# Machine Schemas
+# ============================================================================
+
+
+class MachineSchema(Schema):
+    """Schema for Machine model."""
+
+    TYPE: str = "machine.Machine"
+    id: UUID
+    created_at: datetime
+    modified_at: datetime
+    guid: str
+    hostname: str
+    hw_in_docker: bool
+    hw_in_vm: bool
+    hw_manufacturer: str
+    hw_product: str
+    hw_uuid: str
+    os_arch: str
+    os_family: str
+    os_platform: str
+    os_release: str
+    os_kernel: str
+    stats: dict
+    num_uses_succeeded: int
+    num_uses_failed: int
+
+
+class MachineFilterSchema(FilterSchema):
+    id: Annotated[str | None, FilterLookup("id__startswith")] = None
+    hostname: Annotated[str | None, FilterLookup("hostname__icontains")] = None
+    os_platform: Annotated[str | None, FilterLookup("os_platform__icontains")] = None
+    os_arch: Annotated[str | None, FilterLookup("os_arch")] = None
+    hw_in_docker: Annotated[bool | None, FilterLookup("hw_in_docker")] = None
+    hw_in_vm: Annotated[bool | None, FilterLookup("hw_in_vm")] = None
+    bin_providers: Annotated[str | None, FilterLookup("bin_providers__icontains")] = None
+
+
+# ============================================================================
+# Binary Schemas
+# ============================================================================
+
+
+class BinarySchema(Schema):
+    """Schema for Binary model."""
+
+    TYPE: str = "machine.Binary"
+    id: UUID
+    created_at: datetime
+    modified_at: datetime
+    machine_id: UUID
+    machine_hostname: str
+    name: str
+    binproviders: str
+    binprovider: str
+    abspath: str
+    version: str
+    sha256: str
+    status: str
+    is_valid: bool
+    num_uses_succeeded: int
+    num_uses_failed: int
+
+    @staticmethod
+    def resolve_machine_hostname(obj) -> str:
+        return obj.machine.hostname
+
+    @staticmethod
+    def resolve_is_valid(obj) -> bool:
+        return obj.is_valid
+
+
+class BinaryFilterSchema(FilterSchema):
+    id: Annotated[str | None, FilterLookup("id__startswith")] = None
+    name: Annotated[str | None, FilterLookup("name__icontains")] = None
+    binprovider: Annotated[str | None, FilterLookup("binprovider")] = None
+    status: Annotated[str | None, FilterLookup("status")] = None
+    machine_id: Annotated[str | None, FilterLookup("machine_id__startswith")] = None
+    version: Annotated[str | None, FilterLookup("version__icontains")] = None
+
+
+# ============================================================================
+# Machine Endpoints
+# ============================================================================
+
+
+@router.get("/machines", response=list[MachineSchema], url_name="get_machines")
+@paginate(CustomPagination)
+def get_machines(request: HttpRequest, filters: Query[MachineFilterSchema]):
+    """List all machines."""
+    from archivebox.machine.models import Machine
+
+    return filters.filter(Machine.objects.all()).distinct()
+
+
+@router.get("/machine/current", response=MachineSchema, url_name="get_current_machine")
+def get_current_machine(request: HttpRequest):
+    """Get the current machine."""
+    from archivebox.machine.models import Machine
+
+    return Machine.current()
+
+
+@router.get("/machine/{machine_id}", response=MachineSchema, url_name="get_machine")
+def get_machine(request: HttpRequest, machine_id: str):
+    """Get a specific machine by ID."""
+    from archivebox.machine.models import Machine
+    from django.db.models import Q
+
+    return Machine.objects.get(Q(id__startswith=machine_id) | Q(hostname__iexact=machine_id))
+
+
+# ============================================================================
+
+
+# ============================================================================
+# Binary Endpoints
+# ============================================================================
+
+
+@router.get("/binaries", response=list[BinarySchema], url_name="get_binaries")
+@paginate(CustomPagination)
+def get_binaries(request: HttpRequest, filters: Query[BinaryFilterSchema]):
+    """List all binaries."""
+    from archivebox.machine.models import Binary
+
+    return filters.filter(Binary.objects.all().select_related("machine")).distinct()
+
+
+@router.get("/binary/{binary_id}", response=BinarySchema, url_name="get_binary")
+def get_binary(request: HttpRequest, binary_id: str):
+    """Get a specific binary by ID."""
+    from archivebox.machine.models import Binary
+
+    return Binary.objects.select_related("machine").get(id__startswith=binary_id)
+
+
+@router.get("/binary/by-name/{name}", response=list[BinarySchema], url_name="get_binaries_by_name")
+def get_binaries_by_name(request: HttpRequest, name: str):
+    """Get all binaries with the given name."""
+    from archivebox.machine.models import Binary
+
+    return list(Binary.objects.filter(name__iexact=name).select_related("machine"))
@@ -0,0 +1,157 @@
+__package__ = "archivebox.api"
+
+import json
+from datetime import datetime
+from typing import Any
+from uuid import UUID
+
+from django.db.models import Q
+from django.http import HttpRequest
+from ninja import Router, Schema
+from pydantic import Field
+
+from archivebox.personas.importers import validate_persona_name
+from archivebox.personas.models import Persona
+
+
+router = Router(tags=["Personas"])
+
+
+class PersonaBrowserSettingsSchema(Schema):
+    user_agent: str = ""
+    viewport_size: str = ""
+    viewport_device_scale_factor: float | None = None
+    language: str = ""
+    timezone: str = ""
+    geolocation: dict[str, Any] | None = None
+
+
+class PersonaSyncSchema(Schema):
+    extension_persona_id: str
+    name: str
+    settings: PersonaBrowserSettingsSchema = Field(default_factory=PersonaBrowserSettingsSchema)
+    cookies_txt: str = ""
+    auth_json: dict[str, Any] = Field(default_factory=dict)
+
+
+class PersonaSchema(Schema):
+    TYPE: str = "personas.models.Persona"
+    id: UUID
+    name: str
+    created_at: datetime
+    created_by_id: str
+    created_by_username: str
+    config: dict[str, Any] | None
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by.pk)
+
+    @staticmethod
+    def resolve_created_by_username(obj) -> str:
+        return obj.created_by.username
+
+
+class PersonaSyncResponseSchema(Schema):
+    success: bool
+    created: bool
+    persona: PersonaSchema
+    cookies_file_written: bool
+    auth_file_written: bool
+
+
+def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrowserSettingsSchema) -> dict[str, Any]:
+    config: dict[str, Any] = {
+        "BROWSER_EXTENSION_PERSONA_ID": extension_persona_id,
+        "BROWSER_EXTENSION_SYNCED_AT": datetime.utcnow().isoformat() + "Z",
+    }
+
+    if settings.user_agent:
+        config.update(
+            {
+                "USER_AGENT": settings.user_agent,
+                "CHROME_USER_AGENT": settings.user_agent,
+                "WGET_USER_AGENT": settings.user_agent,
+                "CURL_USER_AGENT": settings.user_agent,
+            },
+        )
+    if settings.viewport_size:
+        config.update(
+            {
+                "RESOLUTION": settings.viewport_size,
+                "CHROME_RESOLUTION": settings.viewport_size,
+            },
+        )
+    if settings.viewport_device_scale_factor is not None:
+        config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
+    if settings.language:
+        config["BROWSER_LANGUAGE"] = settings.language
+    if settings.timezone:
+        config["BROWSER_TIMEZONE"] = settings.timezone
+    if settings.geolocation:
+        config["BROWSER_GEOLOCATION"] = settings.geolocation
+
+    return config
+
+
+def find_persona(extension_persona_id: str, name: str) -> Persona | None:
+    return (
+        Persona.objects.filter(
+            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
+        )
+        .order_by("created_at")
+        .first()
+    )
+
+
+@router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
+def get_personas(request: HttpRequest):
+    """List personas available on this ArchiveBox server."""
+    return Persona.objects.all().order_by("name")
+
+
+@router.post("/sync", response=PersonaSyncResponseSchema, url_name="sync_persona")
+def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
+    """
+    Create or update a Persona from a browser extension profile export.
+
+    The extension sends browser settings plus portable auth artifacts. The server
+    keeps browser override settings in Persona.config and writes cookies.txt /
+    auth.json into the persona directory for extractors to consume.
+    """
+    name = payload.name.strip()
+    is_valid, error_message = validate_persona_name(name)
+    if not is_valid:
+        raise ValueError(error_message)
+
+    persona = find_persona(payload.extension_persona_id, name)
+    created = persona is None
+    if persona is None:
+        persona = Persona(name=name)
+        if getattr(request.user, "is_authenticated", False):
+            persona.created_by = request.user
+
+    persona.config = {
+        **(persona.config or {}),
+        **browser_settings_to_config(payload.extension_persona_id, payload.settings),
+    }
+    persona.save()
+    persona.ensure_dirs()
+
+    cookies_written = False
+    if payload.cookies_txt.strip():
+        (persona.path / "cookies.txt").write_text(payload.cookies_txt)
+        cookies_written = True
+
+    auth_written = False
+    if payload.auth_json:
+        (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
+        auth_written = True
+
+    return {
+        "success": True,
+        "created": created,
+        "persona": persona,
+        "cookies_file_written": cookies_written,
+        "auth_file_written": auth_written,
+    }
@@ -0,0 +1,32 @@
+__package__ = "archivebox.api"
+
+from typing import Any
+from collections.abc import Callable
+import logging
+
+from django.db import transaction
+from signal_webhooks.handlers import sync_task_handler
+
+
+logger = logging.getLogger(__name__)
+
+
+def warning_error_handler(hook: Any, error: Exception | None) -> None:
+    if error is not None:
+        logger.warning("Outbound webhook %r failed: %s", hook.name, error)
+        return
+
+    logger.warning("Outbound webhook %r returned a non-success response.", hook.name)
+
+
+def transaction_on_commit_task_handler(hook: Callable[..., None], **kwargs: Any) -> None:
+    def run_webhook() -> None:
+        try:
+            sync_task_handler(hook, **kwargs)
+        except Exception:
+            logger.warning("Outbound webhook failed after transaction commit.", exc_info=True)
+
+    try:
+        transaction.on_commit(run_webhook)
+    except Exception:
+        logger.warning("Could not schedule outbound webhook after transaction commit.", exc_info=True)
@@ -0,0 +1 @@
+__package__ = "archivebox.base_models"
@@ -0,0 +1,743 @@
+"""Base admin classes for models using UUIDv7."""
+
+__package__ = "archivebox.base_models"
+
+import json
+from collections.abc import Mapping
+from typing import NotRequired, TypedDict
+
+from django import forms
+from django.contrib import admin
+from django.db import models
+from django.forms.renderers import BaseRenderer
+from django.http import HttpRequest, QueryDict
+from django.utils.safestring import SafeString, mark_safe
+from django_object_actions import DjangoObjectActions
+
+
+class ConfigOption(TypedDict):
+    plugin: str
+    type: str | list[str]
+    default: object
+    description: str
+    enum: NotRequired[list[object]]
+    pattern: NotRequired[str]
+    minimum: NotRequired[int | float]
+    maximum: NotRequired[int | float]
+
+
+class KeyValueWidget(forms.Widget):
+    """
+    A widget that renders JSON dict as editable key-value input fields
+    with + and - buttons to add/remove rows.
+    Includes autocomplete for available config keys from the plugin system.
+    """
+
+    template_name = ""  # We render manually
+
+    class Media:
+        css = {
+            "all": [],
+        }
+        js = []
+
+    def _get_config_options(self) -> dict[str, ConfigOption]:
+        """Get available config options from plugins."""
+        try:
+            from archivebox.config.common import ArchiveBoxConfig
+            from archivebox.hooks import discover_plugin_configs
+
+            options: dict[str, ConfigOption] = {}
+            skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "CRAWL_OUTPUT_DIR", "SNAP_DIR"}
+            for key, field in ArchiveBoxConfig.model_fields.items():
+                if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
+                    continue
+                default = field.default
+                try:
+                    json.dumps(default)
+                except TypeError:
+                    default = str(default)
+                options[key] = {
+                    "plugin": "archivebox",
+                    "type": str(field.annotation),
+                    "default": default,
+                    "description": field.description or "",
+                }
+
+            plugin_configs = discover_plugin_configs()
+            for plugin_name, schema in plugin_configs.items():
+                for key, prop in schema.get("properties", {}).items():
+                    option: ConfigOption = {
+                        "plugin": plugin_name,
+                        "type": prop.get("type", "string"),
+                        "default": prop.get("default", ""),
+                        "description": prop.get("description", ""),
+                    }
+                    for schema_key in ("enum", "pattern", "minimum", "maximum"):
+                        if schema_key in prop:
+                            option[schema_key] = prop[schema_key]
+                    options[key] = option
+            return options
+        except Exception:
+            return {}
+
+    def _parse_value(self, value: object) -> dict[str, object]:
+        # Parse JSON value to dict
+        if value is None:
+            return {}
+        if isinstance(value, str):
+            try:
+                parsed = json.loads(value) if value else {}
+            except json.JSONDecodeError:
+                return {}
+            return parsed if isinstance(parsed, dict) else {}
+        if isinstance(value, Mapping):
+            return {str(key): item for key, item in value.items()}
+        return {}
+
+    def render(
+        self,
+        name: str,
+        value: object,
+        attrs: Mapping[str, str] | None = None,
+        renderer: BaseRenderer | None = None,
+    ) -> SafeString:
+        data = self._parse_value(value)
+
+        widget_id = attrs.get("id", name) if attrs else name
+        config_options = self._get_config_options()
+
+        # Build datalist options
+        datalist_options = "\n".join(
+            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
+            for key, opt in sorted(config_options.items())
+        )
+
+        # Build config metadata as JSON for JS
+        config_meta_json = json.dumps(config_options)
+
+        html = f'''
+        <div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
+            <datalist id="{widget_id}_keys">
+                {datalist_options}
+            </datalist>
+            <div id="{widget_id}_rows" class="key-value-rows">
+        '''
+
+        # Render existing key-value pairs
+        for key, val in data.items():
+            val_str = json.dumps(val) if not isinstance(val, str) else val
+            html += self._render_row(widget_id, key, val_str)
+
+        # Always add one empty row for new entries
+        html += self._render_row(widget_id, "", "")
+
+        html += f'''
+            </div>
+            <div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
+                <button type="button" onclick="addKeyValueRow_{widget_id}()"
+                        style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
+                    + Add Row
+                </button>
+            </div>
+            <input type="hidden" name="{name}" id="{widget_id}" value="">
+            <script>
+                (function() {{
+                    var configMeta_{widget_id} = {config_meta_json};
+                    var rowCounter_{widget_id} = 0;
+
+                    function stringifyValue_{widget_id}(value) {{
+                        return typeof value === 'string' ? value : JSON.stringify(value);
+                    }}
+
+                    function getTypes_{widget_id}(meta) {{
+                        if (!meta || meta.type === undefined || meta.type === null) {{
+                            return [];
+                        }}
+                        return Array.isArray(meta.type) ? meta.type : [meta.type];
+                    }}
+
+                    function getMetaForKey_{widget_id}(key) {{
+                        if (!key) {{
+                            return null;
+                        }}
+
+                        var explicitMeta = configMeta_{widget_id}[key];
+                        if (explicitMeta) {{
+                            return Object.assign({{ key: key }}, explicitMeta);
+                        }}
+
+                        if (key.endsWith('_BINARY')) {{
+                            return {{
+                                key: key,
+                                plugin: 'custom',
+                                type: 'string',
+                                default: '',
+                                description: 'Path to binary executable',
+                            }};
+                        }}
+
+                        if (isRegexConfigKey_{widget_id}(key)) {{
+                            return {{
+                                key: key,
+                                plugin: 'custom',
+                                type: 'string',
+                                default: '',
+                                description: 'Regex pattern list',
+                            }};
+                        }}
+
+                        return null;
+                    }}
+
+                    function describeMeta_{widget_id}(meta) {{
+                        if (!meta) {{
+                            return '';
+                        }}
+
+                        var details = '';
+                        if (Array.isArray(meta.enum) && meta.enum.length) {{
+                            details = 'Allowed: ' + meta.enum.map(stringifyValue_{widget_id}).join(', ');
+                        }} else {{
+                            var types = getTypes_{widget_id}(meta);
+                            if (types.length) {{
+                                details = 'Expected: ' + types.join(' or ');
+                            }}
+                        }}
+
+                        if (meta.minimum !== undefined || meta.maximum !== undefined) {{
+                            var bounds = [];
+                            if (meta.minimum !== undefined) bounds.push('min ' + meta.minimum);
+                            if (meta.maximum !== undefined) bounds.push('max ' + meta.maximum);
+                            details += (details ? ' ' : '') + '(' + bounds.join(', ') + ')';
+                        }}
+
+                        return [meta.description || '', details].filter(Boolean).join(' ');
+                    }}
+
+                    function getExampleInput_{widget_id}(key, meta) {{
+                        var types = getTypes_{widget_id}(meta);
+                        if (key.endsWith('_BINARY')) {{
+                            return 'Example: wget or /usr/bin/wget';
+                        }}
+                        if (key.endsWith('_ARGS_EXTRA') || key.endsWith('_ARGS')) {{
+                            return 'Example: ["--extra-arg"]';
+                        }}
+                        if (types.includes('array')) {{
+                            return 'Example: ["value"]';
+                        }}
+                        if (types.includes('object')) {{
+                            if (key === 'SAVE_ALLOWLIST' || key === 'SAVE_DENYLIST') {{
+                                return 'Example: {{"^https://example\\\\.com": ["wget"]}}';
+                            }}
+                            return 'Example: {{"key": "value"}}';
+                        }}
+                        return '';
+                    }}
+
+                    function isRegexConfigKey_{widget_id}(key) {{
+                        return key === 'URL_ALLOWLIST' ||
+                            key === 'URL_DENYLIST' ||
+                            key === 'SAVE_ALLOWLIST' ||
+                            key === 'SAVE_DENYLIST' ||
+                            key.endsWith('_PATTERN') ||
+                            key.includes('REGEX');
+                    }}
+
+                    function isSimpleFilterPattern_{widget_id}(pattern) {{
+                        return /^[\\w.*:-]+$/.test(pattern);
+                    }}
+
+                    function validateRegexPattern_{widget_id}(pattern) {{
+                        if (!pattern || isSimpleFilterPattern_{widget_id}(pattern)) {{
+                            return '';
+                        }}
+
+                        try {{
+                            new RegExp(pattern);
+                        }} catch (error) {{
+                            return error && error.message ? error.message : 'Invalid regex';
+                        }}
+                        return '';
+                    }}
+
+                    function validateRegexConfig_{widget_id}(key, raw, typeName) {{
+                        if (typeName === 'object') {{
+                            var parsed;
+                            try {{
+                                parsed = JSON.parse(raw);
+                            }} catch (error) {{
+                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
+                            }}
+                            if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {{
+                                return {{ ok: false, value: parsed, message: 'Must be a JSON object' }};
+                            }}
+                            for (var regexKey in parsed) {{
+                                var objectRegexError = validateRegexPattern_{widget_id}(regexKey);
+                                if (objectRegexError) {{
+                                    return {{ ok: false, value: parsed, message: 'Invalid regex key "' + regexKey + '": ' + objectRegexError }};
+                                }}
+                            }}
+                            return {{ ok: true, value: parsed, message: '' }};
+                        }}
+
+                        var patterns = raw.split(/[\\n,]+/).map(function(pattern) {{
+                            return pattern.trim();
+                        }}).filter(Boolean);
+                        for (var i = 0; i < patterns.length; i++) {{
+                            var regexError = validateRegexPattern_{widget_id}(patterns[i]);
+                            if (regexError) {{
+                                return {{ ok: false, value: raw, message: 'Invalid regex "' + patterns[i] + '": ' + regexError }};
+                            }}
+                        }}
+                        return {{ ok: true, value: raw, message: '' }};
+                    }}
+
+                    function validateBinaryValue_{widget_id}(raw) {{
+                        if (!raw) {{
+                            return {{ ok: true, value: raw, message: '' }};
+                        }}
+
+                        if (/['"`]/.test(raw)) {{
+                            return {{ ok: false, value: raw, message: 'Binary paths cannot contain quotes' }};
+                        }}
+
+                        if (/[;&|<>$(){{}}\\[\\]!]/.test(raw)) {{
+                            return {{ ok: false, value: raw, message: 'Binary paths can only be a binary name or absolute path' }};
+                        }}
+
+                        if (raw.startsWith('/')) {{
+                            if (/^[A-Za-z0-9_./+\\- ]+$/.test(raw)) {{
+                                return {{ ok: true, value: raw, message: '' }};
+                            }}
+                            return {{ ok: false, value: raw, message: 'Absolute paths may only contain path-safe characters' }};
+                        }}
+
+                        if (/^[A-Za-z0-9_.+-]+$/.test(raw)) {{
+                            return {{ ok: true, value: raw, message: '' }};
+                        }}
+
+                        return {{ ok: false, value: raw, message: 'Enter a binary name like wget or an absolute path like /usr/bin/wget' }};
+                    }}
+
+                    function parseValue_{widget_id}(raw) {{
+                        try {{
+                            if (raw === 'true') return true;
+                            if (raw === 'false') return false;
+                            if (raw === 'null') return null;
+                            if (raw !== '' && !isNaN(raw)) return Number(raw);
+                            if ((raw.startsWith('{{') && raw.endsWith('}}')) ||
+                                (raw.startsWith('[') && raw.endsWith(']')) ||
+                                (raw.startsWith('"') && raw.endsWith('"'))) {{
+                                return JSON.parse(raw);
+                            }}
+                        }} catch (error) {{
+                            return raw;
+                        }}
+                        return raw;
+                    }}
+
+                    function sameValue_{widget_id}(left, right) {{
+                        return left === right || JSON.stringify(left) === JSON.stringify(right);
+                    }}
+
+                    function parseTypedValue_{widget_id}(raw, typeName, meta) {{
+                        var numberValue;
+                        var parsed;
+
+                        if (typeName && meta && meta.key && isRegexConfigKey_{widget_id}(meta.key)) {{
+                            return validateRegexConfig_{widget_id}(meta.key, raw, typeName);
+                        }}
+
+                        if (typeName === 'string' && meta && meta.key && meta.key.endsWith('_BINARY')) {{
+                            return validateBinaryValue_{widget_id}(raw);
+                        }}
+
+                        if (typeName === 'string') {{
+                            if (meta.pattern) {{
+                                try {{
+                                    if (!(new RegExp(meta.pattern)).test(raw)) {{
+                                        return {{ ok: false, value: raw, message: 'Must match pattern ' + meta.pattern }};
+                                    }}
+                                }} catch (error) {{}}
+                            }}
+                            return {{ ok: true, value: raw, message: '' }};
+                        }}
+
+                        if (typeName === 'integer') {{
+                            if (!/^-?\\d+$/.test(raw)) {{
+                                return {{ ok: false, value: raw, message: 'Must be an integer' }};
+                            }}
+                            numberValue = Number(raw);
+                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
+                            }}
+                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
+                            }}
+                            return {{ ok: true, value: numberValue, message: '' }};
+                        }}
+
+                        if (typeName === 'number') {{
+                            if (raw === '' || isNaN(raw)) {{
+                                return {{ ok: false, value: raw, message: 'Must be a number' }};
+                            }}
+                            numberValue = Number(raw);
+                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
+                            }}
+                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
+                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
+                            }}
+                            return {{ ok: true, value: numberValue, message: '' }};
+                        }}
+
+                        if (typeName === 'boolean') {{
+                            var lowered = raw.toLowerCase();
+                            if (lowered === 'true' || raw === '1') return {{ ok: true, value: true, message: '' }};
+                            if (lowered === 'false' || raw === '0') return {{ ok: true, value: false, message: '' }};
+                            return {{ ok: false, value: raw, message: 'Must be true or false' }};
+                        }}
+
+                        if (typeName === 'null') {{
+                            return raw === 'null'
+                                ? {{ ok: true, value: null, message: '' }}
+                                : {{ ok: false, value: raw, message: 'Must be null' }};
+                        }}
+
+                        if (typeName === 'array' || typeName === 'object') {{
+                            try {{
+                                parsed = JSON.parse(raw);
+                            }} catch (error) {{
+                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
+                            }}
+
+                            if (typeName === 'array' && Array.isArray(parsed)) {{
+                                return {{ ok: true, value: parsed, message: '' }};
+                            }}
+                            if (typeName === 'object' && parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {{
+                                return {{ ok: true, value: parsed, message: '' }};
+                            }}
+
+                            return {{
+                                ok: false,
+                                value: parsed,
+                                message: typeName === 'array' ? 'Must be a JSON array' : 'Must be a JSON object',
+                            }};
+                        }}
+
+                        return {{ ok: true, value: parseValue_{widget_id}(raw), message: '' }};
+                    }}
+
+                    function validateValueAgainstMeta_{widget_id}(raw, meta) {{
+                        if (!meta || raw === '') {{
+                            return {{ state: 'neutral', value: raw, message: '' }};
+                        }}
+
+                        var enumValues = Array.isArray(meta.enum) ? meta.enum : [];
+                        var types = getTypes_{widget_id}(meta);
+                        if (!types.length) {{
+                            types = ['string'];
+                        }}
+
+                        var error = 'Invalid value';
+                        for (var i = 0; i < types.length; i++) {{
+                            var candidate = parseTypedValue_{widget_id}(raw, types[i], meta);
+                            if (!candidate.ok) {{
+                                error = candidate.message || error;
+                                continue;
+                            }}
+                            if (enumValues.length && !enumValues.some(function(enumValue) {{
+                                return sameValue_{widget_id}(enumValue, candidate.value) || stringifyValue_{widget_id}(enumValue) === raw;
+                            }})) {{
+                                error = 'Must be one of: ' + enumValues.map(stringifyValue_{widget_id}).join(', ');
+                                continue;
+                            }}
+                            return {{ state: 'valid', value: candidate.value, message: '' }};
+                        }}
+
+                        return {{ state: 'invalid', value: raw, message: error }};
+                    }}
+
+                    function ensureRowId_{widget_id}(row) {{
+                        if (!row.dataset.rowId) {{
+                            row.dataset.rowId = String(rowCounter_{widget_id}++);
+                        }}
+                        return row.dataset.rowId;
+                    }}
+
+                    function setRowHelp_{widget_id}(row) {{
+                        var keyInput = row.querySelector('.kv-key');
+                        var help = row.querySelector('.kv-help');
+                        if (!keyInput || !help) {{
+                            return;
+                        }}
+
+                        var key = keyInput.value.trim();
+                        if (!key) {{
+                            help.textContent = '';
+                            return;
+                        }}
+
+                        var meta = getMetaForKey_{widget_id}(key);
+                        if (meta) {{
+                            var extra = isRegexConfigKey_{widget_id}(key)
+                                ? ((meta.type === 'object' || (Array.isArray(meta.type) && meta.type.includes('object')))
+                                    ? ' Expected: JSON object with regex keys.'
+                                    : ' Expected: valid regex.')
+                                : '';
+                            var example = getExampleInput_{widget_id}(key, meta);
+                            help.textContent = [describeMeta_{widget_id}(meta) + extra, example].filter(Boolean).join(' ');
+                        }} else {{
+                            help.textContent = 'Custom key';
+                        }}
+                    }}
+
+                    function configureValueInput_{widget_id}(row) {{
+                        var keyInput = row.querySelector('.kv-key');
+                        var valueInput = row.querySelector('.kv-value');
+                        var datalist = row.querySelector('.kv-value-options');
+                        if (!keyInput || !valueInput || !datalist) {{
+                            return;
+                        }}
+
+                        var rowId = ensureRowId_{widget_id}(row);
+                        datalist.id = '{widget_id}_value_options_' + rowId;
+
+                        var meta = getMetaForKey_{widget_id}(keyInput.value.trim());
+                        var enumValues = Array.isArray(meta && meta.enum) ? meta.enum : [];
+                        var types = getTypes_{widget_id}(meta);
+                        if (!enumValues.length && types.includes('boolean')) {{
+                            enumValues = ['True', 'False'];
+                        }}
+                        if (enumValues.length) {{
+                            datalist.innerHTML = enumValues.map(function(enumValue) {{
+                                return '<option value="' + stringifyValue_{widget_id}(enumValue).replace(/"/g, '&quot;') + '"></option>';
+                            }}).join('');
+                            valueInput.setAttribute('list', datalist.id);
+                        }} else {{
+                            datalist.innerHTML = '';
+                            valueInput.removeAttribute('list');
+                        }}
+                    }}
+
+                    function setValueValidationState_{widget_id}(input, state, message) {{
+                        if (!input) {{
+                            return;
+                        }}
+
+                        if (state === 'valid') {{
+                            input.style.borderColor = '#2da44e';
+                            input.style.boxShadow = '0 0 0 1px rgba(45, 164, 78, 0.18)';
+                            input.style.backgroundColor = '#f6ffed';
+                        }} else if (state === 'invalid') {{
+                            input.style.borderColor = '#cf222e';
+                            input.style.boxShadow = '0 0 0 1px rgba(207, 34, 46, 0.18)';
+                            input.style.backgroundColor = '#fff8f8';
+                        }} else {{
+                            input.style.borderColor = '#ccc';
+                            input.style.boxShadow = 'none';
+                            input.style.backgroundColor = '';
+                        }}
+                        input.title = message || '';
+                    }}
+
+                    function applyValueValidation_{widget_id}(row) {{
+                        var keyInput = row.querySelector('.kv-key');
+                        var valueInput = row.querySelector('.kv-value');
+                        if (!keyInput || !valueInput) {{
+                            return;
+                        }}
+
+                        var key = keyInput.value.trim();
+                        if (!key) {{
+                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
+                            return;
+                        }}
+
+                        var meta = getMetaForKey_{widget_id}(key);
+                        if (!meta) {{
+                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
+                            return;
+                        }}
+
+                        var validation = validateValueAgainstMeta_{widget_id}(valueInput.value.trim(), meta);
+                        setValueValidationState_{widget_id}(valueInput, validation.state, validation.message);
+                    }}
+
+                    function coerceValueForStorage_{widget_id}(key, raw) {{
+                        var meta = getMetaForKey_{widget_id}(key);
+                        if (!meta) {{
+                            return parseValue_{widget_id}(raw);
+                        }}
+
+                        var validation = validateValueAgainstMeta_{widget_id}(raw, meta);
+                        return validation.state === 'valid' ? validation.value : raw;
+                    }}
+
+                    function initializeRows_{widget_id}() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
+                            ensureRowId_{widget_id}(row);
+                            configureValueInput_{widget_id}(row);
+                            setRowHelp_{widget_id}(row);
+                            applyValueValidation_{widget_id}(row);
+                        }});
+                    }}
+
+                    function updateHiddenField_{widget_id}() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        var rows = container.querySelectorAll('.key-value-row');
+                        var result = {{}};
+                        rows.forEach(function(row) {{
+                            var keyInput = row.querySelector('.kv-key');
+                            var valInput = row.querySelector('.kv-value');
+                            if (keyInput && valInput && keyInput.value.trim()) {{
+                                var key = keyInput.value.trim();
+                                var val = valInput.value.trim();
+                                result[key] = coerceValueForStorage_{widget_id}(key, val);
+                            }}
+                        }});
+                        document.getElementById('{widget_id}').value = JSON.stringify(result);
+                    }}
+
+                    window.addKeyValueRow_{widget_id} = function() {{
+                        var container = document.getElementById('{widget_id}_rows');
+                        var newRow = document.createElement('div');
+                        newRow.className = 'key-value-row';
+                        newRow.style.cssText = 'margin-bottom: 6px;';
+                        newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
+                            '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
+                            'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
+                            '<input type="text" class="kv-value" placeholder="value" ' +
+                            'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
+                            '<datalist class="kv-value-options"></datalist>' +
+                            '<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
+                            'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>' +
+                            '</div>' +
+                            '<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>';
+                        container.appendChild(newRow);
+                        ensureRowId_{widget_id}(newRow);
+                        configureValueInput_{widget_id}(newRow);
+                        setRowHelp_{widget_id}(newRow);
+                        applyValueValidation_{widget_id}(newRow);
+                        updateHiddenField_{widget_id}();
+                        newRow.querySelector('.kv-key').focus();
+                    }};
+
+                    window.removeKeyValueRow_{widget_id} = function(btn) {{
+                        var row = btn.closest('.key-value-row');
+                        row.remove();
+                        updateHiddenField_{widget_id}();
+                    }};
+
+                    window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
+
+                    // Initialize on load
+                    document.addEventListener('DOMContentLoaded', function() {{
+                        initializeRows_{widget_id}();
+                        updateHiddenField_{widget_id}();
+                    }});
+                    // Also run immediately in case DOM is already ready
+                    if (document.readyState !== 'loading') {{
+                        initializeRows_{widget_id}();
+                        updateHiddenField_{widget_id}();
+                    }}
+
+                    // Update on any input change
+                    var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
+
+                    rowsEl_{widget_id}.addEventListener('input', function(event) {{
+                        var row = event.target.closest('.key-value-row');
+                        if (!row) {{
+                            return;
+                        }}
+
+                        if (event.target.classList.contains('kv-key')) {{
+                            configureValueInput_{widget_id}(row);
+                            setRowHelp_{widget_id}(row);
+                        }}
+
+                        if (event.target.classList.contains('kv-key') || event.target.classList.contains('kv-value')) {{
+                            applyValueValidation_{widget_id}(row);
+                            updateHiddenField_{widget_id}();
+                        }}
+                    }});
+                }})();
+            </script>
+        </div>
+        '''
+        return mark_safe(html)
+
+    def _render_row(self, widget_id: str, key: str, value: str) -> str:
+        return f'''
+            <div class="key-value-row" style="margin-bottom: 6px;">
+                <div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
+                    <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
+                           style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
+                    <input type="text" class="kv-value" value="{self._escape(value)}" placeholder="value"
+                           style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
+                    <datalist class="kv-value-options"></datalist>
+                    <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
+                            style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>
+                </div>
+                <div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>
+            </div>
+        '''
+
+    def _escape(self, s: object) -> str:
+        """Escape HTML special chars in attribute values."""
+        if not s:
+            return ""
+        return str(s).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
+
+    def value_from_datadict(
+        self,
+        data: QueryDict | Mapping[str, object],
+        files: object,
+        name: str,
+    ) -> str:
+        value = data.get(name, "{}")
+        return value if isinstance(value, str) else "{}"
+
+
+class ConfigEditorMixin(admin.ModelAdmin):
+    """
+    Mixin for admin classes with a config JSON field.
+
+    Provides a key-value editor widget with autocomplete for available config keys.
+    """
+
+    def formfield_for_dbfield(
+        self,
+        db_field: models.Field,
+        request: HttpRequest,
+        **kwargs: object,
+    ) -> forms.Field | None:
+        """Use KeyValueWidget for the config JSON field."""
+        if db_field.name == "config":
+            kwargs["widget"] = KeyValueWidget()
+        return super().formfield_for_dbfield(db_field, request, **kwargs)
+
+
+class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
+    list_display = ("id", "created_at", "created_by")
+    readonly_fields = ("id", "created_at", "modified_at")
+    show_search_mode_selector = False
+
+    def get_default_search_mode(self) -> str:
+        # The shared changelist template always asks every admin for a default
+        # search mode, even when the search-mode toggle is hidden.
+        return "meta"
+
+    def get_form(
+        self,
+        request: HttpRequest,
+        obj: models.Model | None = None,
+        change: bool = False,
+        **kwargs: object,
+    ):
+        form = super().get_form(request, obj, change=change, **kwargs)
+        if "created_by" in form.base_fields:
+            form.base_fields["created_by"].initial = request.user
+        return form
@@ -0,0 +1,7 @@
+# from django.apps import AppConfig
+
+
+# class BaseModelsConfig(AppConfig):
+#     default_auto_field = 'django.db.models.BigAutoField'
+
+#     name = 'base_models'
@@ -0,0 +1,269 @@
+"""Base models using UUIDv7 for all id fields."""
+
+__package__ = "archivebox.base_models"
+
+import shutil
+
+from archivebox.uuid_compat import uuid7
+from pathlib import Path
+
+from django.db import models
+from django.db.models import F
+from django.db import transaction
+from django.db.models.signals import pre_delete
+from django.utils import timezone
+from django.contrib.auth import get_user_model
+from django.urls import reverse_lazy
+from django.conf import settings
+
+from django_stubs_ext.db.models import TypedModelMeta
+
+
+def get_or_create_system_user_pk(username="system"):
+    User = get_user_model()
+    # If there's exactly one superuser, use that for all system operations
+    if User.objects.filter(is_superuser=True).count() == 1:
+        return User.objects.filter(is_superuser=True).values_list("pk", flat=True)[0]
+    # Otherwise get or create the system user
+    user, _ = User.objects.get_or_create(
+        username=username,
+        defaults={"is_staff": True, "is_superuser": True, "email": "", "password": "!"},
+    )
+    return user.pk
+
+
+class AutoDateTimeField(models.DateTimeField):
+    """DateTimeField that automatically updates on save (legacy compatibility)."""
+
+    def pre_save(self, model_instance, add):
+        if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
+            value = timezone.now()
+            setattr(model_instance, self.attname, value)
+            return value
+        return super().pre_save(model_instance, add)
+
+
+class ModelWithUUID(models.Model):
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
+    modified_at = models.DateTimeField(auto_now=True)
+    created_by = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        default=get_or_create_system_user_pk,
+        null=False,
+        db_index=True,
+    )
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    def __str__(self) -> str:
+        return f"[{self.id}] {self.__class__.__name__}"
+
+    @property
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
+
+    @property
+    def api_url(self) -> str:
+        return str(reverse_lazy("api-1:get_any", args=[self.id]))
+
+    @property
+    def api_docs_url(self) -> str:
+        return f"/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}"
+
+
+class ModelWithNotes(models.Model):
+    """Mixin for models with a notes field."""
+
+    notes = models.TextField(blank=True, null=False, default="")
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+
+class ModelWithHealthStats(models.Model):
+    """Mixin for models with health tracking fields."""
+
+    num_uses_failed = models.PositiveIntegerField(default=0)
+    num_uses_succeeded = models.PositiveIntegerField(default=0)
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    @property
+    def health(self) -> int:
+        total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
+        return round((self.num_uses_succeeded / total) * 100)
+
+    def increment_health_stats(self, success: bool):
+        """Atomically increment success or failure counter using F() expression."""
+        field = "num_uses_succeeded" if success else "num_uses_failed"
+        type(self).objects.filter(pk=self.pk).update(
+            **{
+                field: F(field) + 1,
+                "modified_at": timezone.now(),
+            },
+        )
+
+
+class ModelWithConfig(models.Model):
+    """Mixin for models with a JSON config field."""
+
+    config = models.JSONField(default=dict, null=True, blank=True, editable=True)
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+
+class ModelWithDeleteAfter(models.Model):
+    delete_after_final_statuses: tuple[str, ...] = ()
+    delete_at = models.DateTimeField(default=None, null=True, blank=True, db_index=True)
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    def save(self, *args, **kwargs):
+        update_fields = kwargs.get("update_fields")
+        if self.delete_at is None:
+            self.set_delete_at_from_config()
+            if self.delete_at is not None and update_fields is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
+        super().save(*args, **kwargs)
+
+    def get_delete_after_config_value(self):
+        from archivebox.config.common import get_config
+
+        return get_config(include_machine=False).DELETE_AFTER
+
+    def set_delete_at_from_config(self, config_value=None) -> bool:
+        if self.delete_at is not None:
+            return False
+
+        from archivebox.config.common import parse_delete_after
+
+        duration = parse_delete_after(self.get_delete_after_config_value() if config_value is None else config_value)
+        if duration is None:
+            return False
+
+        self.delete_at = (self.created_at or timezone.now()) + duration
+        return True
+
+    @classmethod
+    def missing_delete_at_candidates(cls):
+        return cls.objects.none()
+
+    @classmethod
+    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
+        if backfill_missing:
+            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
+            for obj in missing_delete_at:
+                if obj.set_delete_at_from_config():
+                    cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
+                        delete_at=obj.delete_at,
+                        modified_at=timezone.now(),
+                    )
+
+        # Keep the expiration sweep anchored on delete_at. Some large tables
+        # have millions of final-status rows but almost no retained rows; adding
+        # the status filter before SQLite has narrowed by delete_at can make the
+        # runner scan the hot status index before it claims new work.
+        due_pks = list(
+            cls.objects.filter(delete_at__isnull=False, delete_at__lte=timezone.now())
+            .order_by("delete_at", "pk")
+            .values_list("pk", flat=True)[:batch_size],
+        )
+        if not due_pks:
+            return 0
+
+        queryset = cls.objects.filter(pk__in=due_pks)
+        if cls.delete_after_final_statuses:
+            queryset = queryset.filter(status__in=cls.delete_after_final_statuses)
+
+        count = 0
+        expired = list(queryset.order_by("delete_at", "pk"))
+        for obj in expired:
+            obj.delete()
+            count += 1
+        return count
+
+
+class ModelWithOutputDir(ModelWithUUID):
+    class Meta(ModelWithUUID.Meta):
+        abstract = True
+
+    _delete_signal_registered = False
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        output_dir = Path(self.output_dir)
+        # Avoid holding SQLite write transactions open across slow filesystem work.
+        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
+        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
+
+    @property
+    def output_dir_parent(self) -> str:
+        return f"{self._meta.model_name}s"
+
+    @property
+    def output_dir_name(self) -> str:
+        return str(self.id)
+
+    @property
+    def output_dir_str(self) -> str:
+        return f"{self.output_dir_parent}/{self.output_dir_name}"
+
+    @property
+    def output_dir(self) -> Path:
+        raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
+
+    def output_paths_for_delete(self) -> tuple[Path, ...]:
+        return (Path(self.output_dir),)
+
+    @classmethod
+    def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
+        from archivebox.config.common import get_config
+
+        data_dir = get_config().DATA_DIR.resolve()
+        safe_paths = []
+        for raw_path in paths:
+            path = Path(raw_path)
+            is_safe = False
+            for candidate in (path.absolute(), path.resolve()):
+                try:
+                    candidate.relative_to(data_dir)
+                    is_safe = True
+                    break
+                except ValueError:
+                    continue
+            if not is_safe:
+                raise ValueError(f"Refusing to delete output path outside DATA_DIR: {path}")
+            safe_paths.append(path)
+        return tuple(safe_paths)
+
+    @classmethod
+    def delete_output_paths(cls, paths) -> None:
+        for path in cls.validate_output_paths_for_delete(paths):
+            if path.is_symlink() or path.is_file():
+                path.unlink(missing_ok=True)
+            elif path.is_dir():
+                shutil.rmtree(path, ignore_errors=True)
+
+    @classmethod
+    def register_delete_signal(cls) -> None:
+        if cls._delete_signal_registered:
+            return
+
+        def schedule_output_dir_cleanup(sender, instance, **kwargs):
+            if not isinstance(instance, ModelWithOutputDir):
+                return
+            paths = instance.validate_output_paths_for_delete(instance.output_paths_for_delete())
+            transaction.on_commit(lambda paths=paths: instance.delete_output_paths(paths))
+
+        pre_delete.connect(
+            schedule_output_dir_cleanup,
+            dispatch_uid="archivebox.output_dir_cleanup_on_delete",
+            weak=False,
+        )
+        cls._delete_signal_registered = True
@@ -1,259 +1,219 @@
-__package__ = 'archivebox.cli'
-__command__ = 'archivebox'
-
+__package__ = "archivebox.cli"
+__command__ = "archivebox"
 import os
 import sys
-import argparse
-import threading
-
-from time import sleep
-from collections.abc import Mapping
-
-from rich import print
-
-from typing import Optional, List, IO, Union, Iterable
-from pathlib import Path
-
 from importlib import import_module

-BUILTIN_LIST = list
+import rich_click as click
+from rich.console import Console

-CLI_DIR = Path(__file__).resolve().parent
+from archivebox.config.version import VERSION

-# rewrite setup -> install for backwards compatibility
-if len(sys.argv) > 1 and sys.argv[1] == 'setup':
-    from rich import print
-    print(':warning: [bold red]DEPRECATED[/bold red] `archivebox setup` is deprecated, use `archivebox install` instead')
-    sys.argv[1] = 'install'
-
-if '--debug' in sys.argv:
-    os.environ['DEBUG'] = 'True'
-    sys.argv.remove('--debug')
+STDERR = Console(stderr=True)


-# def list_subcommands() -> Dict[str, str]:
-#     """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
-#     COMMANDS = []
-#     for filename in os.listdir(CLI_DIR):
-#         if is_cli_module(filename):
-#             subcommand = filename.replace('archivebox_', '').replace('.py', '')
-#             module = import_module('.archivebox_{}'.format(subcommand), __package__)
-#             assert is_valid_cli_module(module, subcommand)
-#             COMMANDS.append((subcommand, module.main.__doc__))
-#             globals()[subcommand] = module.main
-#     display_order = lambda cmd: (
-#         display_first.index(cmd[0])
-#         if cmd[0] in display_first else
-#         100 + len(cmd[0])
-#     )
-#     return dict(sorted(COMMANDS, key=display_order))
-
-# just define it statically, it's much faster:
-SUBCOMMAND_MODULES = {
-    'help': 'archivebox_help',
-    'version': 'archivebox_version' ,
-    
-    'init': 'archivebox_init',
-    'install': 'archivebox_install',
-    ##############################################
-    'config': 'archivebox_config',
-    'add': 'archivebox_add',
-    'remove': 'archivebox_remove',
-    'update': 'archivebox_update',
-    'list': 'archivebox_list',
-    'status': 'archivebox_status',
-    
-    'schedule': 'archivebox_schedule',
-    'server': 'archivebox_server',
-    'shell': 'archivebox_shell',
-    'manage': 'archivebox_manage',
-
-    # 'oneshot': 'archivebox_oneshot',
-}
-
-# every imported command module must have these properties in order to be valid
-required_attrs = ('__package__', '__command__', 'main')
-
-# basic checks to make sure imported files are valid subcommands
-is_cli_module = lambda fname: fname.startswith('archivebox_') and fname.endswith('.py')
-is_valid_cli_module = lambda module, subcommand: (
-    all(hasattr(module, attr) for attr in required_attrs)
-    and module.__command__.split(' ')[-1] == subcommand
-)
-
-class LazySubcommands(Mapping):
-    def keys(self):
-        return SUBCOMMAND_MODULES.keys()
-    
-    def values(self):
-        return [self[key] for key in self.keys()]
-    
-    def items(self):
-        return [(key, self[key]) for key in self.keys()]
-    
-    def __getitem__(self, key):
-        module = import_module(f'.{SUBCOMMAND_MODULES[key]}', __package__)
-        assert is_valid_cli_module(module, key)
-        return module.main
-    
-    def __iter__(self):
-        return iter(SUBCOMMAND_MODULES.keys())
-    
-    def __len__(self):
-        return len(SUBCOMMAND_MODULES)
-
-CLI_SUBCOMMANDS = LazySubcommands()
+if "--debug" in sys.argv:
+    os.environ["DEBUG"] = "True"
+    sys.argv.remove("--debug")


-# these common commands will appear sorted before any others for ease-of-use
-meta_cmds = ('help', 'version')                               # dont require valid data folder at all
-setup_cmds = ('init', 'setup', 'install')                      # require valid data folder, but dont require DB present in it yet
-archive_cmds = ('add', 'remove', 'update', 'list', 'status', 'schedule', 'server', 'shell', 'manage')  # require valid data folder + existing db present
-fake_db = ("oneshot",)                                        # use fake in-memory db
+class ArchiveBoxGroup(click.Group):
+    """lazy loading click group for archivebox commands"""

-display_first = (*meta_cmds, *setup_cmds, *archive_cmds)
+    meta_commands = {
+        "help": "archivebox.cli.archivebox_help.main",
+        "version": "archivebox.cli.archivebox_version.main",
+        "mcp": "archivebox.cli.archivebox_mcp.main",
+    }
+    setup_commands = {
+        "init": "archivebox.cli.archivebox_init.main",
+        "install": "archivebox.cli.archivebox_install.main",
+    }
+    # Model commands (CRUD operations via subcommands)
+    model_commands = {
+        "crawl": "archivebox.cli.archivebox_crawl.main",
+        "snapshot": "archivebox.cli.archivebox_snapshot.main",
+        "archiveresult": "archivebox.cli.archivebox_archiveresult.main",
+        "tag": "archivebox.cli.archivebox_tag.main",
+        "binary": "archivebox.cli.archivebox_binary.main",
+        "process": "archivebox.cli.archivebox_process.main",
+        "machine": "archivebox.cli.archivebox_machine.main",
+        "persona": "archivebox.cli.archivebox_persona.main",
+    }
+    archive_commands = {
+        # High-level commands
+        "add": "archivebox.cli.archivebox_add.main",
+        "extract": "archivebox.cli.archivebox_extract.main",
+        "list": "archivebox.cli.archivebox_list.main",
+        "remove": "archivebox.cli.archivebox_remove.main",
+        "run": "archivebox.cli.archivebox_run.main",
+        "update": "archivebox.cli.archivebox_update.main",
+        "status": "archivebox.cli.archivebox_status.main",
+        "search": "archivebox.cli.archivebox_search.main",
+        "config": "archivebox.cli.archivebox_config.main",
+        "schedule": "archivebox.cli.archivebox_schedule.main",
+        "server": "archivebox.cli.archivebox_server.main",
+        "shell": "archivebox.cli.archivebox_shell.main",
+        "manage": "archivebox.cli.archivebox_manage.main",
+        # Introspection commands
+        "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
+    }
+    legacy_model_commands = {
+        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
+        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
+    }
+    all_subcommands = {
+        **meta_commands,
+        **setup_commands,
+        **model_commands,
+        **archive_commands,
+    }
+    renamed_commands = {
+        "setup": "install",
+        "import": "add",
+        "archive": "add",
+    }
+    legacy_model_subcommands = {
+        "crawl": {"create", "list", "update", "delete"},
+        "snapshot": {"create", "list", "update", "delete"},
+    }
+
+    @classmethod
+    def get_canonical_name(cls, cmd_name):
+        return cls.renamed_commands.get(cmd_name, cmd_name)
+
+    @classmethod
+    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
+        if cmd_name not in cls.legacy_model_commands:
+            return False
+
+        try:
+            arg_idx = sys.argv.index(cmd_name)
+        except ValueError:
+            return False
+
+        remaining_args = sys.argv[arg_idx + 1 :]
+        if not remaining_args:
+            return False
+
+        first_arg = remaining_args[0]
+        if first_arg in ("-h", "--help"):
+            return False
+
+        return first_arg not in cls.legacy_model_subcommands[cmd_name]
+
+    def get_command(self, ctx, cmd_name):
+        # handle renamed commands
+        if cmd_name in self.renamed_commands:
+            new_name = self.renamed_commands[cmd_name]
+            STDERR.print(
+                f" [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`",
+            )
+            cmd_name = new_name
+            ctx.invoked_subcommand = cmd_name
+
+        if self._should_use_legacy_model_command(cmd_name):
+            return self._lazy_load(self.legacy_model_commands[cmd_name])
+
+        # handle lazy loading of commands
+        if cmd_name in self.all_subcommands:
+            return self._lazy_load(cmd_name)
+
+        # fall-back to using click's default command lookup
+        return super().get_command(ctx, cmd_name)
+
+    @classmethod
+    def _lazy_load(cls, cmd_name_or_path):
+        import_path = cls.all_subcommands.get(cmd_name_or_path)
+        if import_path is None:
+            import_path = cmd_name_or_path
+        modname, funcname = import_path.rsplit(".", 1)
+
+        # print(f'LAZY LOADING {import_path}')
+        mod = import_module(modname)
+        func = getattr(mod, funcname)
+
+        if not hasattr(func, "__doc__"):
+            raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
+
+        # if not isinstance(cmd, click.BaseCommand):
+        # raise ValueError(f'lazy loading of {import_path} failed - not a click command')
+
+        return func


-IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread', 'Scheduler')  # threads we dont have to wait for before exiting
+@click.group(cls=ArchiveBoxGroup, invoke_without_command=True)
+@click.option("--help", "-h", is_flag=True, help="Show help")
+@click.version_option(VERSION, "-v", "--version", package_name="archivebox", message="%(version)s")
+@click.pass_context
+def cli(ctx, help=False):
+    """ArchiveBox: The self-hosted internet archive"""
+
+    subcommand = ArchiveBoxGroup.get_canonical_name(ctx.invoked_subcommand)
+
+    # if --help is passed or no subcommand is given, show custom help message
+    if help or ctx.invoked_subcommand is None:
+        ctx.invoke(ctx.command.get_command(ctx, "help"))
+
+    # if the subcommand is in archive_commands or model_commands,
+    # then we need to set up the django environment and check that we're in a valid data folder
+    wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+    if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
+        # print('SETUP DJANGO AND CHECK DATA FOLDER')
+        try:
+            if subcommand == "server":
+                run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes")
+                if run_in_debug:
+                    os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
+                    if "--reload" in sys.argv:
+                        os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
+
+            from archivebox.config.django import setup_django
+            from archivebox.misc.checks import check_data_folder, check_migrations
+
+            setup_django()
+            check_data_folder()
+            if subcommand != "update":
+                check_migrations(auto_apply=True)
+        except Exception as e:
+            STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
+            if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
+                raise


-def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: Iterable[str]=IGNORED_BG_THREADS, timeout: int=60) -> int:
-    """
-    Block until the specified threads exit. e.g. pass thread_names=('default_hook_handler',) to wait for webhooks.
-    Useful for waiting for signal handlers, webhooks, etc. to finish running after a mgmt command completes.
-    """
+def main(args=None, prog_name=None, stdin=None):
+    # show `docker run archivebox xyz` in help messages if running in docker
+    IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
+    IS_TTY = sys.stdin.isatty()
+    prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")

-    wait_for_all: bool = thread_names == ()
+    # stdin param allows passing input data from caller (used by __main__.py)
+    # currently not used by click-based CLI, but kept for backwards compatibility

-    thread_matches = lambda thread, ptns: any(ptn in repr(thread) for ptn in ptns)
+    previous_unraisablehook = sys.unraisablehook

-    should_wait = lambda thread: (
-        not thread_matches(thread, ignore_names)
-        and (wait_for_all or thread_matches(thread, thread_names)))
-
-    for tries in range(timeout):
-        all_threads = [*threading.enumerate()]
-        blocking_threads = [*filter(should_wait, all_threads)]
-        threads_summary = ', '.join(repr(t) for t in blocking_threads)
-        if blocking_threads:
-            sleep(1)
-            if tries == 5:                            # only show stderr message if we need to wait more than 5s
-                print(
-                    f'[…] Waiting up to {timeout}s for background jobs (e.g. webhooks) to finish...',
-                    threads_summary,
-                    file=sys.stderr,
-                )
-        else:
-            return tries
-
-    raise Exception(f'Background threads failed to exit after {tries}s: {threads_summary}')
-
-
-
-def run_subcommand(subcommand: str,
-                   subcommand_args: List[str] | None = None,
-                   stdin: Optional[IO]=None,
-                   pwd: Union[Path, str, None]=None) -> None:
-    """Run a given ArchiveBox subcommand with the given list of args"""
-
-    subcommand_args = subcommand_args or []
-
-    from archivebox.misc.checks import check_migrations
-    from archivebox.config.legacy import setup_django
-    
-    # print('DATA_DIR is', DATA_DIR)
-    # print('pwd is', os.getcwd())    
-
-    cmd_requires_db = subcommand in archive_cmds
-    init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
-
-    check_db = cmd_requires_db and not init_pending
-
-    setup_django(in_memory_db=subcommand in fake_db, check_db=check_db)
-
-    if subcommand in archive_cmds:
-        if cmd_requires_db:
-            check_migrations()
-
-    module = import_module('.archivebox_{}'.format(subcommand), __package__)
-    module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore
-
-    # wait for webhooks, signals, and other background jobs to finish before exit
-    wait_for_bg_threads_to_exit(timeout=60)
-
-
-
-
-
-class NotProvided:
-    def __len__(self):
-        return 0
-    def __bool__(self):
-        return False
-    def __repr__(self):
-        return '<not provided>'
-
-Omitted = Union[None, NotProvided]
-
-OMITTED = NotProvided()
-
-
-def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: str | None=None) -> None:
-    # print('STARTING CLI MAIN ENTRYPOINT')
-    
-    args = sys.argv[1:] if args is OMITTED else args
-    stdin = sys.stdin if stdin is OMITTED else stdin
-
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description='ArchiveBox: The self-hosted internet archive',
-        add_help=False,
-    )
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument(
-        '--help', '-h',
-        action='store_true',
-        help=CLI_SUBCOMMANDS['help'].__doc__,
-    )
-    group.add_argument(
-        '--version',
-        action='store_true',
-        help=CLI_SUBCOMMANDS['version'].__doc__,
-    )
-    group.add_argument(
-        "subcommand",
-        type=str,
-        help= "The name of the subcommand to run",
-        nargs='?',
-        choices=CLI_SUBCOMMANDS.keys(),
-        default=None,
-    )
-    parser.add_argument(
-        "subcommand_args",
-        help="Arguments for the subcommand",
-        nargs=argparse.REMAINDER,
-    )
-    command = parser.parse_args(args or ())
-
-    if command.version:
-        command.subcommand = 'version'
-    elif command.help or command.subcommand is None:
-        command.subcommand = 'help'
-
-    if command.subcommand not in ('version',):
-        from ..logging_util import log_cli_command
-
-        log_cli_command(
-            subcommand=command.subcommand,
-            subcommand_args=command.subcommand_args,
-            stdin=stdin or None,
-        )
+    def ignore_shutdown_unraisable(unraisable):
+        if isinstance(unraisable.exc_value, (KeyboardInterrupt, SystemExit)):
+            return
+        previous_unraisablehook(unraisable)

+    sys.unraisablehook = ignore_shutdown_unraisable
    try:
-        run_subcommand(
-            subcommand=command.subcommand,
-            subcommand_args=command.subcommand_args,
-            stdin=stdin or None,
-        )
+        cli(args=args, prog_name=prog_name, standalone_mode=False)
+    except click.Abort:
+        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
+        raise SystemExit(130) from None
+    except click.ClickException as err:
+        err.show()
+        raise SystemExit(err.exit_code) from None
+    except click.exceptions.Exit as err:
+        raise SystemExit(err.exit_code) from None
    except KeyboardInterrupt:
-        print('\n\n[red][X] Got CTRL+C. Exiting...[/red]')
+        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
+        raise SystemExit(130) from None
+    finally:
+        sys.unraisablehook = previous_unraisablehook
+
+
+if __name__ == "__main__":
+    main()
@@ -1,142 +1,352 @@
 #!/usr/bin/env python3
+from __future__ import annotations

-__package__ = 'archivebox.cli'
-__command__ = 'archivebox add'
+__package__ = "archivebox.cli"
+__command__ = "archivebox add"

 import sys
-import argparse
+import json
+import os
+from pathlib import Path

-from typing import List, Optional, IO
+from typing import TYPE_CHECKING

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from archivebox.config.common import ARCHIVING_CONFIG
+import rich_click as click

-from ..main import add
-from ..parsers import PARSERS
-from ..logging_util import SmartFormatter, accept_stdin, stderr
+from archivebox.misc.util import enforce_types, docstring
+from archivebox.misc.util import parse_filesize_to_bytes


+if TYPE_CHECKING:
+    from django.db.models import QuerySet
+    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
+
+
+def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
+    from archivebox.misc.jsonl import read_args_or_stdin
+
+    urls: list[str] = []
+    for record in read_args_or_stdin(args):
+        url = record.get("url")
+        if isinstance(url, str) and url:
+            urls.append(url)
+
+        urls_field = record.get("urls")
+        if isinstance(urls_field, str):
+            for line in urls_field.splitlines():
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    urls.append(line)
+
+    return urls
+
+
+@enforce_types
+def add(
+    urls: str | list[str],
+    snapshot_ids: list[str] | None = None,
+    depth: int | str = 0,
+    max_urls: int = 0,
+    crawl_max_size: int | str = 0,
+    crawl_timeout: int = 0,
+    snapshot_max_size: int | str = 0,
+    crawl_max_concurrent_snapshots: int | None = None,
+    tag: str = "",
+    url_allowlist: str = "",
+    url_denylist: str = "",
+    parser: str = "auto",
+    plugins: str = "",
+    persona: str = "Default",
+    overwrite: bool = False,
+    update: bool | None = None,
+    index_only: bool = False,
+    bg: bool = False,
+    created_by_id: int | None = None,
+) -> tuple[Crawl, QuerySet[Snapshot]]:
+    """Add a new URL or list of URLs to your archive.
+
+    The flow is:
+    1. Save URLs to sources file
+    2. Create Crawl with URLs and max_depth
+    3. Crawl runner creates Snapshots from Crawl URLs (depth=0)
+    4. Crawl runner runs parser extractors on root snapshots
+    5. Parser extractors output to urls.jsonl
+    6. URLs are added to Crawl.urls and child Snapshots are created
+    7. Repeat until max_depth is reached
+    """
+
+    from rich import print
+
+    depth = int(depth)
+    max_urls = int(max_urls or 0)
+    crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
+    crawl_timeout = int(crawl_timeout or 0)
+    snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
+    from archivebox import CONSTANTS
+    from archivebox.config.permissions import USER, HOSTNAME
+    from archivebox.config.common import get_config
+
+    config = get_config()
+    crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
+    if crawl_max_concurrent_snapshots is None:
+        crawl_max_concurrent_snapshots = config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
+    crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
+
+    if depth not in (0, 1, 2, 3, 4):
+        raise ValueError("Depth must be 0-4")
+    if max_urls < 0:
+        raise ValueError("max_urls must be >= 0")
+    if crawl_max_size < 0:
+        raise ValueError("crawl_max_size must be >= 0")
+    if crawl_timeout < 0:
+        raise ValueError("crawl_timeout must be >= 0")
+    if snapshot_max_size < 0:
+        raise ValueError("snapshot_max_size must be >= 0")
+    if crawl_max_concurrent_snapshots < 1:
+        raise ValueError("crawl_max_concurrent_snapshots must be >= 1")
+
+    # import models once django is set up
+    from archivebox.crawls.models import Crawl
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.personas.models import Persona
+    from archivebox.misc.logging_util import printable_filesize
+    from archivebox.misc.system import get_dir_size
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from django.contrib.auth import get_user_model
+    from django.utils import timezone
+
+    created_by_id = created_by_id or get_or_create_system_user_pk()
+    created_by = get_user_model().objects.filter(pk=created_by_id).first()
+    started_at = timezone.now()
+    if update is None:
+        update = not config.ONLY_NEW
+
+    if isinstance(urls, str):
+        url_list = [line.strip() for line in urls.splitlines() if line.strip()]
+    else:
+        url_list = [str(url).strip() for url in urls if str(url).strip()]
+    if snapshot_ids and len(snapshot_ids) != len(url_list):
+        raise ValueError("snapshot_ids length must match urls length")
+
+    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
+    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
+    sources_file.parent.mkdir(parents=True, exist_ok=True)
+    if snapshot_ids:
+        sources_file.write_text(
+            "\n".join(json.dumps({"url": url, "id": snapshot_ids[index], "tags": tag, "depth": 0}) for index, url in enumerate(url_list)),
+        )
+    else:
+        sources_file.write_text("\n".join(url_list))
+
+    # 2. Create a new Crawl with inline URLs
+    cli_args = [*sys.argv]
+    if cli_args[0].lower().endswith("archivebox"):
+        cli_args[0] = "archivebox"
+    cmd_str = " ".join(cli_args)
+
+    timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
+
+    # Read URLs directly into crawl
+    urls_content = sources_file.read_text()
+    persona_name = (persona or "Default").strip() or "Default"
+    plugins = plugins or ""
+    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
+    persona_obj.ensure_dirs()
+    effective_persona_config = get_config(persona=persona_obj, user=created_by)
+
+    crawl_config = {
+        "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
+        **({"ONLY_NEW": not update} if bool(not update) != bool(effective_persona_config.ONLY_NEW) else {}),
+        **({"INDEX_ONLY": True} if index_only else {}),
+        **({"OVERWRITE": True} if overwrite else {}),
+        **({"PLUGINS": plugins} if plugins else {}),
+        **(
+            {"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots}
+            if crawl_max_concurrent_snapshots_override
+            and crawl_max_concurrent_snapshots != int(effective_persona_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS)
+            else {}
+        ),
+        **({"CRAWL_MAX_URLS": max_urls} if max_urls else {}),
+        **({"CRAWL_MAX_SIZE": crawl_max_size} if crawl_max_size else {}),
+        **({"CRAWL_TIMEOUT": crawl_timeout} if crawl_timeout else {}),
+        **({"SNAPSHOT_MAX_SIZE": snapshot_max_size} if snapshot_max_size else {}),
+        **({"PARSER": parser} if parser != "auto" else {}),
+        **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
+        **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
+    }
+
+    crawl = Crawl.objects.create(
+        urls=urls_content,
+        max_depth=depth,
+        tags_str=tag,
+        persona_id=persona_obj.id,
+        label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
+        created_by_id=created_by_id,
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=None if index_only else timezone.now(),
+        config=crawl_config,
+    )
+
+    print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
+    first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
+    print(f"    [dim]First URL: {first_url}[/dim]")
+
+    # 3. The CrawlMachine will create Snapshots from all URLs when started
+    #    Parser extractors run on snapshots and discover more URLs
+    #    Discovered URLs become child Snapshots (depth+1)
+
+    if index_only:
+        print("[yellow]\\[*] Index-only mode - crawl queued, runner not started[/yellow]")
+        return crawl, crawl.snapshot_set.all()
+
+    # 5. Start the crawl runner to process the queue
+    #    The runner will:
+    #    - Process Crawl -> create Snapshots from all URLs
+    #    - Process Snapshots -> run extractors
+    #    - Parser extractors discover new URLs -> create child Snapshots
+    #    - Repeat until max_depth reached
+
+    if bg:
+        # Background mode: just queue work and return (background runner via server will pick it up)
+        print(
+            "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
+        )
+    else:
+        # Foreground mode: run full crawl runner until all work is done
+        print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
+        from archivebox.machine.models import Process
+        from archivebox.services.supervision_service import command_owns_runtime_stack, current_command, standby_until_runtime_stack_needed
+        from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
+
+        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
+        exit_code = 0
+        try:
+            try:
+                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
+                    while True:
+                        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                        exit_code = run_runner_worker(
+                            ["--crawl-id", str(crawl.id)],
+                            name=f"worker_runner_add_{os.getpid()}",
+                            interactive_interrupts=True,
+                        )
+                        if exit_code == 0:
+                            break
+                        if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+                            continue
+                        raise SystemExit(exit_code)
+            except KeyboardInterrupt:
+                exit_code = 130
+                print("\n[red][X] archivebox add interrupted.[/red]")
+                print("[yellow]Hint: resume this crawl with:[/yellow]")
+                print(f"    [green]archivebox run --crawl-id={crawl.id}[/green]")
+                raise SystemExit(exit_code)
+        finally:
+            command.mark_exited(exit_code=exit_code)
+            stop_own_supervisord_process()
+
+        # Print summary for foreground runs
+        try:
+            crawl.refresh_from_db()
+            snapshots_count = crawl.snapshot_set.count()
+            try:
+                from django.db.models import Count, Sum
+
+                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("archiveresult__output_size"))
+                total_bytes = int(totals["total_bytes"] or 0) if totals["snapshot_count"] else 0
+            except Exception:
+                total_bytes, _, _ = get_dir_size(crawl.output_dir)
+            total_size = printable_filesize(total_bytes)
+            total_time = timezone.now() - started_at
+            total_seconds = int(total_time.total_seconds())
+            mins, secs = divmod(total_seconds, 60)
+            hours, mins = divmod(mins, 60)
+            if hours:
+                duration_str = f"{hours}h {mins}m {secs}s"
+            elif mins:
+                duration_str = f"{mins}m {secs}s"
+            else:
+                duration_str = f"{secs}s"
+
+            # Output dir relative to DATA_DIR
+            try:
+                rel_output = Path(crawl.output_dir).relative_to(CONSTANTS.DATA_DIR)
+                rel_output_str = f"./{rel_output}"
+            except Exception:
+                rel_output_str = str(crawl.output_dir)
+
+            from archivebox.core.host_utils import build_admin_url
+
+            admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
+
+            print("\n[bold]crawl output saved to:[/bold]")
+            print(f"  {rel_output_str}")
+            print(f"  {admin_url}")
+            print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
+            print(f"[bold]total size:[/bold] {total_size}")
+            print(f"[bold]total time:[/bold] {duration_str}")
+        except Exception:
+            # Summary is best-effort; avoid failing the command if something goes wrong
+            pass
+
+    # 6. Return the list of Snapshots in this crawl
+    snapshots = crawl.snapshot_set.all()
+    return crawl, snapshots
+
+
+@click.command()
+@click.option(
+    "--depth",
+    "-d",
+    type=click.Choice([str(i) for i in range(5)]),
+    default="0",
+    help="Recursively archive linked pages up to N hops away",
+)
+@click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
+@click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
+@click.option("--crawl-timeout", type=int, default=0, help="Maximum total crawl runtime in seconds (0 = unlimited)")
+@click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)")
+@click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl")
+@click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
+@click.option("--url-allowlist", "--domain-allowlist", default="", help="Comma-separated URL/domain allowlist for this crawl")
+@click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
+@click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
+@click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
+@click.option("--persona", default="Default", help="Authentication profile to use when archiving")
+@click.option("--overwrite", "-F", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
+@click.option("--update", is_flag=True, default=None, help="Retry any previously skipped/failed URLs when re-adding them")
+@click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
+@click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
+@click.argument("urls", nargs=-1, type=click.Path())
@docstring(add.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=add.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        '--tag', '-t',
-        type=str,
-        default='',
-        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
-    )
-    parser.add_argument(
-        '--update', #'-u',
-        action='store_true',
-        default=not ARCHIVING_CONFIG.ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
-        help="Also retry previously skipped/failed links when adding new links",
-    )
-    parser.add_argument(
-        '--update-all', #'-n',
-        action='store_true',
-        default=False, 
-        help="Also update ALL links in index when finished adding new links",
-    )
-    parser.add_argument(
-        '--index-only', #'-o',
-        action='store_true',
-        help="Add the links to the main index without archiving them",
-    )
-    parser.add_argument(
-        'urls',
-        nargs='*',
-        type=str,
-        default=None,
-        help=(
-            'URLs or paths to archive e.g.:\n'
-            '    https://getpocket.com/users/USERNAME/feed/all\n'
-            '    https://example.com/some/rss/feed.xml\n'
-            '    https://example.com\n'
-            '    ~/Downloads/firefox_bookmarks_export.html\n'
-            '    ~/Desktop/sites_list.csv\n'
-        )
-    )
-    parser.add_argument(
-        "--depth",
-        action="store",
-        default=0,
-        choices=[0, 1],
-        type=int,
-        help="Recursively archive all linked pages up to this many hops away"
-    )
-    parser.add_argument(
-        "--overwrite",
-        default=False,
-        action="store_true",
-        help="Re-archive URLs from scratch, overwriting any existing files"
-    )
-    parser.add_argument(
-        "--init", #'-i',
-        action='store_true',
-        help="Init/upgrade the curent data directory before adding",
-    )
-    parser.add_argument(
-        "--extract",
-        type=str,
-        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
-              This does not take precedence over the configuration",
-        default=""
-    )
-    parser.add_argument(
-        "--parser",
-        type=str,
-        help="Parser used to read inputted URLs.",
-        default="auto",
-        choices=["auto", *PARSERS.keys()],
-    )
-    command = parser.parse_args(args or ())
-    urls = command.urls
+def main(**kwargs):
+    """Add a new URL or list of URLs to your archive"""

-    stdin_urls = ''
-    if not urls:
-        stdin_urls = accept_stdin(stdin)
+    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals

-    if (stdin_urls and urls) or (not stdin and not urls):
-        stderr(
-            '[X] You must pass URLs/paths to add via stdin or CLI arguments.\n',
-            color='red',
-        )
-        raise SystemExit(2)
-    add(
-        urls=stdin_urls or urls,
-        depth=command.depth,
-        tag=command.tag,
-        update=command.update,
-        update_all=command.update_all,
-        index_only=command.index_only,
-        overwrite=command.overwrite,
-        init=command.init,
-        extractors=command.extract,
-        parser=command.parser,
-        out_dir=pwd or DATA_DIR,
-    )
+    with foreground_shutdown_signals(), foreground_parent_watchdog():
+        raw_urls = kwargs.pop("urls")
+        urls = _collect_input_urls(raw_urls)
+        if not urls:
+            raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
+        if int(kwargs.get("max_urls") or 0) < 0:
+            raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
+        if int(kwargs.get("crawl_timeout") or 0) < 0:
+            raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout")
+        try:
+            kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
+        except ValueError as err:
+            raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
+        try:
+            kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
+        except ValueError as err:
+            raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
+        if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
+            raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
+
+        add(urls=urls, **kwargs)


-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
-
-
-# TODO: Implement these
-#
-# parser.add_argument(
-#     '--mirror', #'-m',
-#     action='store_true',
-#     help='Archive an entire site (finding all linked pages below it on the same domain)',
-# )
-# parser.add_argument(
-#     '--crawler', #'-r',
-#     choices=('depth_first', 'breadth_first'),
-#     help='Controls which crawler to use in order to find outlinks in a given page',
-#     default=None,
-# )
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,387 @@
+#!/usr/bin/env python3
+
+"""
+archivebox archiveresult <action> [args...] [--filters]
+
+Manage ArchiveResult records (plugin extraction results).
+
+Actions:
+    create  - Create ArchiveResults for Snapshots (queue extractions)
+    list    - List ArchiveResults as JSONL (with optional filters)
+    update  - Update ArchiveResults from stdin JSONL
+    delete  - Delete ArchiveResults from stdin JSONL
+
+Examples:
+    # Create ArchiveResults for snapshots (queue for extraction)
+    archivebox snapshot list --status=queued | archivebox archiveresult create
+    archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
+
+    # List with filters
+    archivebox archiveresult list --status=failed
+    archivebox archiveresult list --plugin=screenshot --status=succeeded
+
+    # Update (reset failed extractions to queued)
+    archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued
+
+    # Delete
+    archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
+
+    # Re-run failed extractions
+    archivebox archiveresult list --status=failed | archivebox run
+"""
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox archiveresult"
+
+import sys
+
+import rich_click as click
+from rich import print as rprint
+
+from archivebox.cli.cli_utils import apply_filters
+
+
+def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
+    return {
+        "type": "ArchiveResult",
+        "snapshot_id": str(snapshot_id),
+        "plugin": plugin,
+        "hook_name": hook_name,
+        "status": status,
+    }
+
+
+# =============================================================================
+# CREATE
+# =============================================================================
+
+
+def create_archiveresults(
+    snapshot_id: str | None = None,
+    plugin: str | None = None,
+    status: str = "queued",
+) -> int:
+    """
+    Create ArchiveResult request records for Snapshots.
+
+    Reads Snapshot records from stdin and emits ArchiveResult request JSONL.
+    Pass-through: Non-Snapshot/ArchiveResult records are output unchanged.
+    If --plugin is specified, only emits requests for that plugin.
+    Otherwise, emits requests for all enabled snapshot hooks.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
+    from archivebox.config.common import get_config
+    from archivebox.hooks import discover_hooks
+    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
+    from archivebox.core.models import Snapshot
+
+    is_tty = sys.stdout.isatty()
+
+    # If snapshot_id provided directly, use that
+    if snapshot_id:
+        try:
+            snapshots = [Snapshot.objects.get(id=snapshot_id)]
+            pass_through_records = []
+        except Snapshot.DoesNotExist:
+            rprint(f"[red]Snapshot not found: {snapshot_id}[/red]", file=sys.stderr)
+            return 1
+    else:
+        # Read from stdin
+        records = list(read_stdin())
+        if not records:
+            rprint("[yellow]No Snapshot records provided via stdin[/yellow]", file=sys.stderr)
+            return 1
+
+        # Separate snapshot records from pass-through records
+        snapshot_ids = []
+        pass_through_records = []
+
+        for record in records:
+            record_type = record.get("type", "")
+
+            if record_type == TYPE_SNAPSHOT:
+                # Pass through the Snapshot record itself
+                pass_through_records.append(record)
+                if record.get("id"):
+                    snapshot_ids.append(record["id"])
+
+            elif record_type == TYPE_ARCHIVERESULT:
+                # ArchiveResult records: pass through if they have an id
+                if record.get("id"):
+                    pass_through_records.append(record)
+                # If no id, we could create it, but for now just pass through
+                else:
+                    pass_through_records.append(record)
+
+            elif record_type:
+                # Other typed records (Crawl, Tag, etc): pass through
+                pass_through_records.append(record)
+
+            elif record.get("id"):
+                # Untyped record with id - assume it's a snapshot ID
+                snapshot_ids.append(record["id"])
+
+        # Output pass-through records first
+        if not is_tty:
+            for record in pass_through_records:
+                write_record(record)
+
+        if not snapshot_ids:
+            if pass_through_records:
+                rprint(f"[dim]Passed through {len(pass_through_records)} records, no new snapshots to process[/dim]", file=sys.stderr)
+                return 0
+            rprint("[yellow]No valid Snapshot IDs in input[/yellow]", file=sys.stderr)
+            return 1
+
+        snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids))
+
+    if not snapshots:
+        rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
+        return 0 if pass_through_records else 1
+
+    created_count = 0
+    for snapshot in snapshots:
+        if plugin:
+            if not is_tty:
+                write_record(build_archiveresult_request(snapshot.id, plugin, status=status))
+            created_count += 1
+        else:
+            config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
+            hooks = discover_hooks("Snapshot", config=config)
+            for hook_path in hooks:
+                hook_name = hook_path.stem
+                plugin_name = hook_path.parent.name
+                if not is_tty:
+                    write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
+                created_count += 1
+
+    rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# LIST
+# =============================================================================
+
+
+def list_archiveresults(
+    status: str | None = None,
+    plugin: str | None = None,
+    snapshot_id: str | None = None,
+    limit: int | None = None,
+) -> int:
+    """
+    List ArchiveResults as JSONL with optional filters.
+
+    Exit codes:
+        0: Success (even if no results)
+    """
+    from archivebox.misc.jsonl import write_record
+    from archivebox.core.models import ArchiveResult
+
+    is_tty = sys.stdout.isatty()
+
+    queryset = ArchiveResult.objects.all().order_by("-start_ts")
+
+    # Apply filters
+    filter_kwargs = {
+        "status": status,
+        "plugin": plugin,
+        "snapshot_id": snapshot_id,
+    }
+    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
+
+    count = 0
+    for result in queryset:
+        if is_tty:
+            status_color = {
+                "queued": "yellow",
+                "started": "blue",
+                "succeeded": "green",
+                "failed": "red",
+                "skipped": "dim",
+                "noresults": "dim",
+                "backoff": "magenta",
+            }.get(result.status, "dim")
+            rprint(
+                f"[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
+            )
+        else:
+            write_record(result.to_json())
+        count += 1
+
+    rprint(f"[dim]Listed {count} archive results[/dim]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# UPDATE
+# =============================================================================
+
+
+def update_archiveresults(
+    status: str | None = None,
+) -> int:
+    """
+    Update ArchiveResults from stdin JSONL.
+
+    Reads ArchiveResult records from stdin and applies updates.
+    Uses PATCH semantics - only specified fields are updated.
+
+    Exit codes:
+        0: Success
+        1: No input or error
+    """
+    from archivebox.misc.jsonl import read_stdin, write_record
+    from archivebox.core.models import ArchiveResult
+
+    is_tty = sys.stdout.isatty()
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    updated_count = 0
+    for record in records:
+        result_id = record.get("id")
+        if not result_id:
+            continue
+
+        try:
+            result = ArchiveResult.objects.get(id=result_id)
+
+            # Apply updates from CLI flags
+            if status:
+                result.status = status
+
+            result.save()
+            updated_count += 1
+
+            if not is_tty:
+                write_record(result.to_json())
+
+        except ArchiveResult.DoesNotExist:
+            rprint(f"[yellow]ArchiveResult not found: {result_id}[/yellow]", file=sys.stderr)
+            continue
+
+    rprint(f"[green]Updated {updated_count} archive results[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# DELETE
+# =============================================================================
+
+
+def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
+    """
+    Delete ArchiveResults from stdin JSONL.
+
+    Requires --yes flag to confirm deletion.
+
+    Exit codes:
+        0: Success
+        1: No input or missing --yes flag
+    """
+    from archivebox.misc.jsonl import read_stdin
+    from archivebox.core.models import ArchiveResult
+
+    records = list(read_stdin())
+    if not records:
+        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+        return 1
+
+    result_ids = [r.get("id") for r in records if r.get("id")]
+
+    if not result_ids:
+        rprint("[yellow]No valid archive result IDs in input[/yellow]", file=sys.stderr)
+        return 1
+
+    results = ArchiveResult.objects.filter(id__in=result_ids)
+    count = results.count()
+
+    if count == 0:
+        rprint("[yellow]No matching archive results found[/yellow]", file=sys.stderr)
+        return 0
+
+    if dry_run:
+        rprint(f"[yellow]Would delete {count} archive results (dry run)[/yellow]", file=sys.stderr)
+        for result in results[:10]:
+            rprint(f"  [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}", file=sys.stderr)
+        if count > 10:
+            rprint(f"  ... and {count - 10} more", file=sys.stderr)
+        return 0
+
+    if not yes:
+        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+        return 1
+
+    # Perform deletion
+    deleted_count, _ = results.delete()
+    rprint(f"[green]Deleted {deleted_count} archive results[/green]", file=sys.stderr)
+    return 0
+
+
+# =============================================================================
+# CLI Commands
+# =============================================================================
+
+
+@click.group()
+def main():
+    """Manage ArchiveResult records (plugin extraction results)."""
+    pass
+
+
+@main.command("create")
+@click.option("--snapshot-id", help="Snapshot ID to create results for")
+@click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
+@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
+def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
+    """Create ArchiveResults for Snapshots from stdin JSONL."""
+    sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
+
+
+@main.command("list")
+@click.option("--status", "-s", help="Filter by status (queued, started, succeeded, failed, skipped)")
+@click.option("--plugin", "-p", help="Filter by plugin name")
+@click.option("--snapshot-id", help="Filter by snapshot ID")
+@click.option("--limit", "-n", type=int, help="Limit number of results")
+def list_cmd(
+    status: str | None,
+    plugin: str | None,
+    snapshot_id: str | None,
+    limit: int | None,
+):
+    """List ArchiveResults as JSONL."""
+    sys.exit(
+        list_archiveresults(
+            status=status,
+            plugin=plugin,
+            snapshot_id=snapshot_id,
+            limit=limit,
+        ),
+    )
+
+
+@main.command("update")
+@click.option("--status", "-s", help="Set status")
+def update_cmd(status: str | None):
+    """Update ArchiveResults from stdin JSONL."""
+    sys.exit(update_archiveresults(status=status))
+
+
+@main.command("delete")
+@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
+@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
+def delete_cmd(yes: bool, dry_run: bool):
+    """Delete ArchiveResults from stdin JSONL."""
+    sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
+
+
+if __name__ == "__main__":
+    main()