fix iterator missing chunk_size

fix ArchiveResult.output_summary admin formatting
2024-08-26 22:18:10 -07:00 · 2024-08-26 22:17:57 -07:00
819 changed files with 30899 additions and 145651 deletions
@@ -1,5 +0,0 @@
-{
-  "enabledPlugins": {
-    "pyright-lsp@claude-plugins-official": true
-  }
-}
@@ -1,51 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(**)",
-      "Glob(**)",
-      "Grep(**)",
-      "Bash(python -m archivebox:*)",
-      "Bash(ls:*)",
-      "Bash(xargs:*)",
-      "Bash(python -c:*)",
-      "Bash(printf:*)",
-      "Bash(pkill:*)",
-      "Bash(python3:*)",
-      "Bash(sqlite3:*)",
-      "WebFetch(domain:github.com)",
-      "Bash(uv add:*)",
-      "Bash(mkdir:*)",
-      "Bash(chmod:*)",
-      "Bash(python -m forum_dl:*)",
-      "Bash(archivebox manage migrate:*)",
-      "Bash(cat:*)",
-      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
-      "Bash(forum-dl:*)",
-      "Bash(pip uninstall:*)",
-      "Bash(python:*)",
-      "Bash(source .venv/bin/activate)",
-      "Bash(mv:*)",
-      "Bash(echo:*)",
-      "Bash(grep:*)",
-      "WebFetch(domain:python-statemachine.readthedocs.io)",
-      "Bash(./bin/run_plugin_tests.sh:*)",
-      "Bash(done)",
-      "Bash(coverage erase:*)",
-      "Bash(gh api:*)"
-    ]
-  },
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
-            "statusMessage": "Checking working directory..."
-          }
-        ]
-      }
-    ]
-  }
-}
@@ -1,8 +1,5 @@
 .DS_Store
 ._*
-/.heartbeat.json
-/package.json
-/package-lock.json
 *.pyc
 __pycache__/
 .mypy_cache/
@@ -12,34 +9,16 @@ __pycache__/
 .pdm-python
 .eggs/
 .git/
-.vscode/
 !.git/HEAD
 !.git/refs/heads/*

 venv/
 .venv/
 .venv-old/
-.docker_venv/
 .docker-venv/
 node_modules/
-abx-dl/
-abxpkg/
-abx-plugins/
-abxbus/
 chrome/
 chromeprofile/
-chrome_profile/
-lib/
-out/
-users/
-archive/
-crawls/
-snapshots/
-logs/
-archivebox-docker-smoke*/
-archivebox-compose-smoke*/
-core
-*.core

 pdm.dev.lock
 pdm.lock
@@ -52,24 +31,9 @@ deb_dist/
 pip_dist/
 assets/
 docker/
-website/
-typings/

-tmp/
-.tmp/
 data/
 data*/
-
-personas/
-sources/
 output/
 index.sqlite3
 index.sqlite3-wal
-queue.sqlite3
-*.sqlite*
-data.*
-.archivebox_id
-ArchiveBox.conf
-*.stdout
-*.stderr
-*.log
@@ -0,0 +1,2 @@
+**/*.lock
+**/*-lock.json
@@ -1,26 +0,0 @@
-# Read the Docs config for https://docs.archivebox.io
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
-
-version: 2
-
-submodules:
-  include: all
-  recursive: true
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.12"
-    #nodejs: "20"     # not needed unless we need the full archivebox to run while building docs for some reason
-
-sphinx:
-  configuration: docs/conf.py
-
-formats:
-  - pdf
-  - epub
-
-# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-python:
-  install:
-    - requirements: docs/requirements.txt
@@ -1,2 +1,2 @@
 github: ["ArchiveBox", "pirate"]
-custom: ["https://donate.archivebox.io", "https://swag.archivebox.io"]
+custom: ["https://donate.archivebox.io", "https://paypal.me/NicholasSweeting"]
@@ -1,198 +0,0 @@
-name: 🐞 Bug report
-description: Report a bug or error you encountered in ArchiveBox
-title: "Bug: ..."
-assignees:
-  - pirate
-type: 'Bug'
-body:
-  - type: markdown
-    attributes:
-      value: |
-        *Please note:* it is normal to see errors occasionally for some extractors on some URLs (not every extractor will work on every type of page).  
-        Please report archiving errors if you are seeing them *consistently across many URLs* or if they are *preventing you from using ArchiveBox*.
-
-  - type: textarea
-    id: description
-    attributes:
-      label: Provide a screenshot and describe the bug
-      description: |
-        Attach a screenshot and describe what the issue is, what you expected to happen, and if relevant, the *URLs you were trying to archive*.  
-      placeholder: |
-        Got a bunch of 'singlefile was unable to archive this page' errors when trying to archive URLs from this site: https://example.com/xyz ...
-        I also tried to archive the same URLs using `singlefile` directly and some of them worked but not all of them. etc. ...
-    validations:
-      required: true
-
-  - type: textarea
-    id: steps_to_reproduce
-    attributes:
-      label: Steps to reproduce
-      description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
-      render: markdown
-      placeholder: |
-        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
-        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
-        3. Typed 'https://example.com/xyz' into the 'Add URL' input field
-        4. Clicked the 'Add+' button
-        5. Got a 500 error and saw the errors below in terminal
-    validations:
-      required: true
-
-  - type: textarea
-    id: logs
-    attributes:
-      label: Logs or errors
-      description: "Paste any terminal output, logs, or errors (check `data/logs/errors.log` as well)."
-      placeholder: |
-        ╭─────────────────────────────────────────────────────────────────────────────────────────────────────────╮
-        │ [2024-11-02 19:54:28] ArchiveBox v0.8.6rc0: archivebox add https://example.com#1234567                  │
-        ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯
-        
-        [+] [2024-11-02 19:54:29] Adding 1 links to index (crawl depth=0)...
-            > Saved verbatim input to sources/1730577269-import.txt
-            > Parsed 1 URLs from input (Generic TXT)
-            ...
-      render: shell
-    validations:
-      required: false
-
-  - type: textarea
-    id: version
-    attributes:
-      label: ArchiveBox Version
-      description: |
-        **REQUIRED:** Run the `archivebox version` command inside your collection dir and paste the *full output* here (*not just the version number*).  
-        For Docker Compose run: `docker compose run archivebox version`  
-        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
-      render: shell
-      placeholder: |
-        0.8.6
-        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
-        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
-        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
-        
-        Binary Dependencies:
-        √  python     3.14.0     venv_pip   ~/.venv/bin/python
-        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
-        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
-        ...
-    validations:
-      required: true
-
-  - type: dropdown
-    id: install_method
-    validations:
-      required: true
-    attributes:
-      label: How did you install the version of ArchiveBox you are using?
-      multiple: false
-      options:
-        - pip
-        - apt
-        - brew
-        - nix
-        - Docker (or Podman/LXC/K8s/TrueNAS/Proxmox/etc)
-        - Other
-
-  - type: dropdown
-    id: operating_system
-    validations:
-      required: true
-    attributes:
-      label: What operating system are you running on?
-      description: |
-        Please note we are *unable to provide support for Windows users* unless you are using [Docker on Windows](https://github.com/ArchiveBox/archivebox#:~:text=windows%20without%20docker).
-      multiple: false
-      options:
-        - Linux (Ubuntu/Debian/Arch/Alpine/etc.)
-        - macOS (including Docker on macOS)
-        - BSD (FreeBSD/OpenBSD/NetBSD/etc.)
-        - Windows (including WSL, WSL2, Docker Desktop on Windows)
-        - Other
-
-  - type: checkboxes
-    id: filesystem
-    attributes:
-      label: What type of drive are you using to store your ArchiveBox data?
-      description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` OR `data/archive`?
-      options:
-        - label: "some of `data/` is on a local SSD or NVMe drive"
-          required: false
-        - label: "some of `data/` is on a spinning hard drive or external USB drive"
-          required: false
-        - label: "some of `data/` is on a network mount (e.g. NFS/SMB/Ceph/GlusterFS/etc.)"
-          required: false
-        - label: "some of `data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/Google Drive/Dropbox/etc.)"
-          required: false
-  
-
-  - type: textarea
-    id: docker_compose_yml
-    attributes:
-      label: Docker Compose Configuration
-      description: "If using Docker Compose, please share your full `docker-compose.yml` file. If using plain Docker, paste the `docker run ...` command you use."
-      placeholder: |
-        services:
-            archivebox:
-                image: archivebox/archivebox:latest
-                ports:
-                    - 8000:8000
-                volumes:
-                    - ./data:/data
-                environment:
-                    - ADMIN_USERNAME=admin
-                    - ADMIN_PASSWORD=****<redact any passwords>****
-                    - ALLOWED_HOSTS=*
-                    - CSRF_TRUSTED_ORIGINS=https://archivebox.example.com
-                    - PUBLIC_INDEX=True
-                    - PUBLIC_SNAPSHOTS=True
-                    - PUBLIC_ADD_VIEW=False
-                    ...
-
-            archivebox_scheduler:
-                image: archivebox/archivebox:latest
-                command: schedule --foreground --update --every=day
-                environment:
-                    ...
-      
-          ...
-      render: shell
-    validations:
-      required: false
-
-  - type: textarea
-    id: configuration
-    attributes:
-      label: ArchiveBox Configuration
-      description: "Please share your full `data/ArchiveBox.conf` file here."
-      render: shell
-      placeholder: |
-        [SERVER_CONFIG]
-        SECRET_KEY = "*********<redact any secrets/passwords>************"
-
-        WGET_RESTRICT_FILE_NAMES=windows
-        USE_SYSTEM_WGET=true
-        CHECK_SSL_VALIDITY=false
-        ...
-    validations:
-      required: false
-
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying anything suggested here, **we are not responsible for data loss**.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
-          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
-          - 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
-          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
-        
@@ -1,128 +0,0 @@
-name: 💡 Feature or enhancement request
-description: Suggest an idea or improvement for this project
-title: "Feature Request: ..."
-assignees:
-  - pirate
-type: 'Enhancement'
-labels: 'status: idea phase'
-body:
-  - type: dropdown
-    id: suggestion_type
-    validations:
-      required: true
-    attributes:
-      label: "What type of suggestion are you making?"
-      multiple: false
-      options:
-        - New extractor / type of content to save
-        - Proposing a new feature
-        - Modification of existing behavior
-        - Web UI or UX design improvement
-
-  - type: textarea
-    id: current_problem
-    attributes:
-      label: "What is the problem that your feature request solves?"
-      description: |
-        Describe the problem or need that your feature request solves, feel free to include any screenshots or examples.
-      placeholder: |
-        e.g. I need to be able to archive spanish and french subtitle files from a particular movie site https://example.com/somevideos that's going down soon.
-    validations:
-      required: true
-
-  - type: textarea
-    id: proposed_solution
-    attributes:
-      label: "What is your proposed solution?"
-      description: |
-        Describe the ideal specific solution you'd want, *and whether it fits into any broader scope of changes*.
-      placeholder: |
-        e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
-        The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
-    validations:
-      required: true
-
-  - type: textarea
-    id: workarounds_tried
-    attributes:
-      label: "What hacks or alternative solutions have you tried to solve the problem?"
-      description: |
-        A description of any alternative approaches, workarounds, or other solutions you've considered to fix the problem.
-      placeholder: |
-        e.g. I wait for archivebox to finish archiving the page, then I manually run `yt-dlp --subs <URL>` inside 
-        the `data/archive/<timestamp>/` directory to download the subtitle files and add them to the snapshot folder.
-    validations:
-      required: true
-
-  - type: textarea
-    id: version
-    attributes:
-      label: Share the entire output of the `archivebox version` command for the current version you are using.
-      description: |
-        DO NOT JUST ENTER "the latest version" OR YOUR ISSUE WILL BE CLOSED.  
-        We need to know what version of ArchiveBox and what feature flags you're currently running with in order to contextualize your feature request.  
-        Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in your specific environment.
-          
-        Run the `archivebox version` command inside your current collection dir and paste the *full output* here (*not just the version number*).  
-        For Docker Compose run: `docker compose run archivebox version`  
-        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
-      render: shell
-      placeholder: |
-        0.8.6
-        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
-        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
-        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
-        
-        Binary Dependencies:
-        √  python     3.14.0     venv_pip   ~/.venv/bin/python
-        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
-        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
-        ...
-    validations:
-      required: true
-
-  - type: checkboxes
-    id: priority
-    attributes:
-      label: "How badly do you want this new feature?"
-      options:
-        - label: "It's an urgent deal-breaker, I can't live without it"
-          required: false
-        - label: "It's important to add it in the near-mid term future"
-          required: false
-        - label: "It would be nice to have eventually"
-          required: false
-        - label: "I'm willing to [start a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
-          required: false
-        - label: "I have [donated money](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations) to go towards fixing this issue"
-          required: false
-  
-  - type: checkboxes
-    id: satisfaction_survey
-    attributes:
-      label: Mini Survey
-      description: How do you like ArchiveBox so far?
-      options:
-        - label: "I like ArchiveBox so far / would recommend it to a friend"
-          required: false
-        - label: "I've had a lot of difficulty getting ArchiveBox set up"
-          required: false
-        - label: "I would pay $10/mo for a hosted version of ArchiveBox if it had this feature"
-          required: false
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying any workarounds or BETAs suggested here, **we are not responsible for data loss**.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 📊 [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
-          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
-          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
@@ -1,52 +0,0 @@
-name: 📑 Documentation improvement
-description: Submit an idea or correction for the Wiki documentation
-title: "Documentation: ..."
-labels: 'touches: docs'
-type: 'Enhancement'
-assignees:
-  - pirate
-body:
-  - type: markdown
-    attributes:
-      value: |
-        If you prefer, you can submit a [Pull Request](https://github.com/ArchiveBox/docs) on https://github.com/ArchiveBox/docs to edit the docs directly instead.
-
-  - type: input
-    id: page_url
-    validations:
-      required: true
-    attributes:
-      label: "What is the URL of the page you'd like to see improved?"
-      placeholder: e.g. https://github.com/ArchiveBox/docs/wiki/Install
-
-  - type: input
-    id: section_title
-    validations:
-      required: true
-    attributes:
-      label: "What is the title of the relevant section?"
-      placeholder: e.g. Option B. Automatic Setup Script
-
-  - type: textarea
-    id: suggested_edit
-    attributes:
-      label: "What is the suggested edit?"
-      placeholder: |
-        e.g. Please document how to run the automatic setup script for ArchiveBox on TempleOS.  
-        Attach images, screenshots, code snippets, etc. anything you think would help.
-    validations:
-      required: true
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to address issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
-          - 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
-          - ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)
@@ -0,0 +1,46 @@
+---
+name: 🐞 Bug report
+about: Create a report to help us improve
+title: 'Bug: ...'
+labels: 'bug'
+assignees: ''
+
+---
+
+<!--
+Please fill out the following information, 
+feel free to delete sections if they're not applicable 
+or if long issue templates annoy you.
+(the only required section is the version information)
+-->
+
+#### Describe the bug
+<!--
+A description of what the bug is, 
+what you expected to happen, 
+and any relevant context about issue.
+-->
+
+#### Steps to reproduce
+<!--
+For example:
+1. Ran ArchiveBox with the following config '...'
+2. Saw this output during archiving '....'
+3. UI didn't show the thing I was expecting '....'
+-->
+
+#### Screenshots or log output
+
+<!--
+If applicable, post any relevant screenshots or copy/pasted terminal output from ArchiveBox.
+If you're reporting a parsing / importing error, **you must paste a copy of your redacted import file here**.
+-->
+
+#### ArchiveBox version
+
+<!-- Run the `archivebox version` command locally then copy paste the result here: -->
+```logs
+replace this line with the *full*, unshortened output of running `archivebox version`
+```
+<!-- Tickets without full version info will closed until it is provided,
+we need the full output here to help you solve your issue -->
@@ -1,11 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: ❓ Ask a question or start a discussion
-    url: https://github.com/ArchiveBox/ArchiveBox/discussions
-    about: "Ask a question, get support, or start a design discussion (to report a problem please use '🐞 Bug report' instead)"
-  - name: 💬 Chat with the dev team & community on Zulip
-    url: https://zulip.archivebox.io
-    about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
-  - name: 💁‍♂️ Hire us for professional support with fast response times
-    url: https://docs.monadical.com/s/archivebox-consulting-services
-    about: "We provide hosting, development, and support, including on-prem/cloud w/ SSO & storage, CAPTCHA-solving, proxies, etc."
@@ -0,0 +1,17 @@
+---
+name: 📑 Documentation change
+about: Submit a suggestion for the Wiki documentation
+title: 'Documentation: Improvement request ...'
+labels: ''
+assignees: ''
+
+---
+<!-- If you perfer, you can make a PR to https://github.com/ArchiveBox/docs instead of opening an issue -->
+
+## Wiki Page URL
+<!-- e.g. https://github.com/pirate/ArchiveBox/wiki/Configuration#use_color -->
+
+
+## Suggested Edit
+<!-- e.g. Please add more example usages, or please fix `xyz` typo to be `abc`. -->
+
@@ -0,0 +1,50 @@
+---
+name: 💡 Feature request
+about: Suggest an idea for this project
+title: 'Feature Request: ...'
+labels: 'changes: behavior,status: idea phase'
+assignees: ''
+
+---
+
+<!--
+Please fill out the following information, 
+feel free to delete sections if they're not applicable 
+or if long issue templates annoy you :)
+-->
+
+## Type 
+
+ - [ ] General question or discussion
+ - [ ] Propose a brand new feature
+ - [ ] Request modification of existing behavior or design
+
+## What is the problem that your feature request solves
+<!--
+e.g. I need to be able to archive spanish and french subtitle files  
+from a particular <example.com> movie site that's going down soon.
+-->
+
+## Describe the ideal specific solution you'd want, and whether it fits into any broader scope of changes
+<!--
+e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
+The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
+-->
+
+## What hacks or alternative solutions have you tried to solve the problem?
+<!--
+A clear and concise description of any alternative solutions, 
+workarounds, or other software you've considered using to fix the problem.
+-->
+
+## How badly do you want this new feature?
+
+ - [ ] It's an urgent deal-breaker, I can't live without it
+ - [ ] It's important to add it in the near-mid term future
+ - [ ] It would be nice to have eventually
+
+---
+
+ - [ ] I'm willing to contribute [dev time](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) / [money](https://github.com/sponsors/pirate) to fix this issue
+ - [ ] I like ArchiveBox so far / would recommend it to a friend
+ - [ ] I've had a lot of difficulty getting ArchiveBox set up
@@ -0,0 +1,9 @@
+---
+name: 💬 Question, discussion, or support request
+about: Start a discussion or ask a question about ArchiveBox
+title: 'Question: ...'
+labels: ''
+assignees: ''
+
+---
+
@@ -1,49 +0,0 @@
-name: Claude Code
-
-on:
-  issue_comment:
-    types: [created]
-  pull_request_review_comment:
-    types: [created]
-  issues:
-    types: [opened, assigned]
-  pull_request_review:
-    types: [submitted]
-
-jobs:
-  claude:
-    if: |
-      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
-      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      pull-requests: write
-      issues: write
-      id-token: write
-      actions: read # Required for Claude to read CI results on PRs
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Run Claude Code
-        id: claude
-        uses: anthropics/claude-code-action@v1
-        with:
-          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
-
-          # This is an optional setting that allows Claude to read CI results on PRs
-          additional_permissions: |
-            actions: read
-
-          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
-          # prompt: 'Update the pull request description to include a summary of changes.'
-
-          # Optional: Add claude_args to customize behavior and configuration
-          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
-          # or https://code.claude.com/docs/en/cli-reference for available options
-          claude_args: '--allowed-tools Bash(gh pr:*)'
@@ -2,209 +2,63 @@ name: Build Debian package

 on:
  workflow_dispatch:
-  workflow_call:
  push:
-    branches: ['**']
-    paths:
-      - 'pkg/debian/**'
-      - 'bin/build_deb.sh'
-      - 'bin/release_deb.sh'
-      - '.github/workflows/debian.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs

-permissions:
-  contents: write
+env:
+  DEB_BUILD_OPTIONS: nocheck

 jobs:
  build:
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        arch: [amd64, arm64]
+    runs-on: ubuntu-22.04

    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
        with:
+          submodules: true
          fetch-depth: 1

-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Install nfpm
-        env:
-          NFPM_VERSION: "2.45.1"
+      - name: Install packaging dependencies
        run: |
-          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
-            | sudo tar -xz -C /usr/local/bin nfpm
-          nfpm --version
+          sudo apt-get update -qq
+          sudo apt-get install -y \
+            python3 python3-dev python3-pip python3-venv python3-all \
+            dh-python debhelper devscripts dput software-properties-common \
+            python3-distutils python3-setuptools python3-wheel python3-stdeb

-      - name: Build .deb package
-        run: |
-          export VERSION="${{ steps.version.outputs.version }}"
-          export ARCH="${{ matrix.arch }}"
-          ./bin/build_deb.sh
+      # - name: Build Debian/Apt sdist_dsc
+      #   run: |
+      #     ./bin/build_pip.sh

-      - name: Verify .deb package contents
-        run: |
-          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
-          echo "=== Package info ==="
-          dpkg-deb --info "$DEB_FILE"
-          echo ""
-          echo "=== Package contents ==="
-          dpkg-deb --contents "$DEB_FILE"
-          echo ""
-          echo "=== Control fields ==="
-          dpkg-deb --field "$DEB_FILE"
+      # - name: Check ArchiveBox version
+      #   run: |
+      #     # must create dir needed for snaps to run as non-root on github actions
+      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
+      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
+      #     archivebox --version
+      #     archivebox init --setup

-      - name: Upload .deb artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
-          path: dist/*.deb
+      # - name: Add some links to test
+      #   run: |
+      #     cd "${{ github.workspace }}/data"
+      #     archivebox add 'https://example.com'
+      #     archivebox status

-  test:
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: ubuntu-24.04
-          - arch: arm64
-            runner: ubuntu-24.04-arm
-    runs-on: ${{ matrix.runner }}
+      # - name: Commit built package
+      #   run: |
+      #     cd deb_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Debian package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/debian-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: deb_dist

-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local wheel
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --wheel --out-dir /tmp/wheels/
-
-      - name: Download .deb artifact
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*-${{ matrix.arch }}.deb
-          merge-multiple: true
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
-
-      - name: Pre-seed virtualenv with local wheel before dpkg install
-        run: |
-          # CI-only: pre-seed the venv with a local wheel so we test the
-          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
-          # here (matching the system dep installed above) to ensure the venv is
-          # created with the correct Python version. On real installs, install.sh
-          # handles this by preferring python3.13 and failing if < 3.13.
-          # When postinstall.sh runs during dpkg -i, it finds the venv already
-          # populated and just upgrades deps.
-          sudo mkdir -p /opt/archivebox
-          sudo python3.13 -m venv /opt/archivebox/venv
-          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
-          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
-          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
-
-      - name: Install .deb package
-        run: |
-          # dpkg install will run postinstall which creates the user,
-          # sets up systemd, and tries pip install (which finds it already installed)
-          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
-
-      - name: Verify archivebox is installed
-        run: |
-          which archivebox
-          archivebox version
-
-      - name: Verify wrapper script works
-        run: |
-          /usr/bin/archivebox version
-          /usr/bin/archivebox --help | head -5
-
-      - name: Test archivebox init as archivebox user
-        run: |
-          # The postinstall should have created the user
-          id archivebox
-          sudo mkdir -p /tmp/archivebox-test
-          sudo chown archivebox:archivebox /tmp/archivebox-test
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
-
-      - name: Test archivebox status
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
-
-      - name: Test archivebox add
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
-
-      - name: Verify systemd service file exists
-        run: |
-          test -f /usr/lib/systemd/system/archivebox.service
-          cat /usr/lib/systemd/system/archivebox.service
-
-  # Upload .deb to GitHub Release only after tests pass
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: [build, test]
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Determine release tag
-        id: tag
-        run: |
-          TAG="${{ github.event.release.tag_name }}"
-          if [ -z "$TAG" ]; then
-            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
-            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
-          fi
-          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-
-      - name: Download all .deb artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*.deb
-          merge-multiple: true
-
-      - name: Upload .deb to GitHub Release
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          TAG="${{ steps.tag.outputs.tag }}"
-          # Verify the release exists before uploading
-          if ! gh release view "$TAG" >/dev/null 2>&1; then
-            echo "[!] No GitHub Release found for tag $TAG."
-            if [ -n "${{ github.event.release.tag_name }}" ]; then
-              echo "[X] This was triggered by a release event — the release should exist. Failing."
-              exit 1
-            fi
-            echo "[i] Skipping upload (workflow_dispatch without a release)."
-            echo "    Create a release first or trigger via the release event."
-            exit 0
-          fi
-          gh release upload "$TAG" *.deb --clobber
+      # - name: Push build to Launchpad PPA
+      #   run: |
+      #     debsign -k "$PGP_KEY_ID" "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
+      #     dput archivebox "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
@@ -1,43 +0,0 @@
-name: Deploy Publicsite to GitHub Pages
-
-on:
-  push:
-    branches:
-      - dev
-    paths:
-      - publicsite/**
-      - .github/workflows/deploy-publicsite.yml
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-concurrency:
-  group: github-pages-publicsite
-  cancel-in-progress: true
-
-jobs:
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Setup Pages
-        uses: actions/configure-pages@v5
-
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
-        with:
-          path: ./publicsite
-
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4
@@ -2,7 +2,6 @@ name: Build Docker image

 on:
  workflow_dispatch:
-  workflow_call:
  push:
    branches:
      - '**'
@@ -19,9 +18,9 @@ jobs:
    steps:
      - name: Checkout
        uses: actions/checkout@v4
-        # with:
-        #   submodules: true
-        #   fetch-depth: 1
+        with:
+          submodules: true
+          fetch-depth: 1

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
@@ -41,34 +40,26 @@ jobs:
        run: echo ${{ steps.buildx.outputs.platforms }}

      - name: Cache Docker layers
-        uses: actions/cache@v4
+        uses: actions/cache@v3
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-

-      - name: Login to Docker Hub
+      - name: Docker Login
        uses: docker/login-action@v3
        if: github.event_name != 'pull_request'
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Collect Full Release Docker tags
+      - name: Collect Docker tags
        # https://github.com/docker/metadata-action
        id: docker_meta
        uses: docker/metadata-action@v5
-        if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
        with:
-          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
+          images: archivebox/archivebox,nikisweeting/archivebox
          tags: |
              # :stable
              type=ref,event=branch
@@ -81,19 +72,6 @@ jobs:
              # :latest
              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}

-      - name: Collect Non-Release Docker tags
-        # https://github.com/docker/metadata-action
-        id: docker_meta_non_release
-        uses: docker/metadata-action@v5
-        if: github.event_name != 'workflow_dispatch' && github.event_name != 'workflow_call'
-        with:
-          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
-          tags: |
-              # :stable
-              type=ref,event=branch
-              # :sha-463ea54
-              type=sha
-
      - name: Build and push
        id: docker_build
        uses: docker/build-push-action@v5
@@ -102,8 +80,7 @@ jobs:
          file: ./Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.tags || steps.docker_meta_non_release.outputs.tags }}
-          labels: ${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.docker_meta.outputs.labels || steps.docker_meta_non_release.outputs.labels }}
+          tags: ${{ steps.docker_meta.outputs.tags }}
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache-new
          platforms: linux/amd64,linux/arm64
@@ -1,59 +0,0 @@
-name: Duplicate Issue Detection
-
-on:
-  issues:
-    types: [opened]
-
-jobs:
-  check-duplicates:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      issues: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Install opencode
-        run: curl -fsSL https://opencode.ai/install | bash
-
-      - name: Check for duplicate issues
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OPENCODE_PERMISSION: |
-            {
-              "bash": {
-                "gh issue*": "allow",
-                "gh pr*": "allow",
-                "*": "deny"
-              },
-              "webfetch": "allow"
-            }
-        run: |
-          opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created:
-
-          Issue number:
-          ${{ github.event.issue.number }}
-
-          Lookup this issue and search through existing issues and PRs (excluding #${{ github.event.issue.number }}) in this repository to find any potential duplicates of this new issue.
-          Consider:
-          1. Similar titles or descriptions
-          2. Same error messages or symptoms
-          3. Related functionality or components
-          4. Similar feature requests
-
-          If you find any potential duplicates, please comment on the new issue with:
-          - A brief explanation of why it might be a duplicate
-          - Links to the potentially duplicate issues or PRs
-          - A suggestion to check those issues first
-
-          Use this format for the comment:
-          'This issue might be a duplicate of existing issues. Please check:
-          - #[issue_number]: [brief description of similarity]
-
-          Feel free to ignore if none of these address your specific case.'
-
-          If no clear duplicates are found, do not comment."
@@ -1,229 +1,51 @@
-name: Build Homebrew formula
+name: Build Homebrew package

 on:
  workflow_dispatch:
-  workflow_call:
  push:
-    branches: ['**']
-    paths:
-      - 'brew_dist/**'
-      - 'bin/build_brew.sh'
-      - 'bin/release_brew.sh'
-      - '.github/workflows/homebrew.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs

-permissions:
-  contents: read

 jobs:
-  build-and-test:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-latest, ubuntu-24.04]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Validate formula template syntax
-        run: ruby -c brew_dist/archivebox.rb
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies (Linux)
-        if: runner.os == 'Linux'
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local sdist
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --sdist --out-dir /tmp/sdist/
-
-      - name: Generate formula from local sdist
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
-          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
-
-          # Install archivebox + poet into a temp venv to generate resource stanzas
-          python3 -m venv /tmp/poet-venv
-          source /tmp/poet-venv/bin/activate
-          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
-          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
-          RESOURCES="$(poet archivebox)"
-          deactivate
-
-          # For CI: use file:// URL pointing to local sdist
-          # For release: this gets overridden with the PyPI URL
-          SDIST_URL="file://${SDIST_PATH}"
-
-          cat > /tmp/archivebox.rb << RUBY
-class Archivebox < Formula
-  include Language::Python::Virtualenv
-
-  desc "Self-hosted internet archiving solution"
-  homepage "https://github.com/ArchiveBox/ArchiveBox"
-  url "${SDIST_URL}"
-  sha256 "${SDIST_SHA256}"
-  license "MIT"
-
-  depends_on "python@3.13"
-
-${RESOURCES}
-
-  def install
-    virtualenv_install_with_resources
-  end
-
-  def post_install
-    data_dir = var/"archivebox"
-    data_dir.mkpath
-    ENV["DATA_DIR"] = data_dir.to_s
-    system bin/"archivebox", "init"
-  end
-
-  def caveats
-    <<~EOS
-      ArchiveBox data is stored in:
-        #{var}/archivebox
-
-      To start archiving, run:
-        cd #{var}/archivebox && archivebox add 'https://example.com'
-
-      To start the web UI:
-        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
-    EOS
-  end
-
-  test do
-    assert_match version.to_s, shell_output("#{bin}/archivebox version")
-  end
-end
-RUBY
-
-          echo "[√] Generated formula:"
-          ruby -c /tmp/archivebox.rb
-          cat /tmp/archivebox.rb
-
-      - name: Install Homebrew (Linux only)
-        if: runner.os == 'Linux'
-        run: |
-          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
-          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
-          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
-          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
-          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
-          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
-
-      - name: Install brew dependencies
-        run: brew install python@3.13
-
-      - name: Install archivebox via brew from local formula
-        run: |
-          brew install --build-from-source --verbose /tmp/archivebox.rb
-
-      - name: Verify archivebox version
-        run: archivebox version
-
-      - name: Test archivebox init
-        run: |
-          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
-          archivebox init
-
-      - name: Test archivebox status
-        run: |
-          cd /tmp/archivebox-test
-          archivebox status
-
-  # On release only: generate the real formula with PyPI URL and push to tap
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: build-and-test
+  build:
    runs-on: macos-latest

    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
        with:
+          submodules: true
          fetch-depth: 1

-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
+      # TODO: modify archivebox.rb to update src url, hashes, and dependencies

-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Wait for PyPI package availability
+      - name: Build Homebrew Bottle
        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
-          for i in $(seq 1 30); do
-            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
-              echo "[√] archivebox==${VERSION} is available on PyPI"
-              break
-            fi
-            if [ "$i" -eq 30 ]; then
-              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
-              break
-            fi
-            echo "    Attempt $i/30 - not yet available, waiting 30s..."
-            sleep 30
-          done
-
-      - name: Generate release formula via build_brew.sh
-        run: ./bin/build_brew.sh
-
-      - name: Test formula install
-        run: |
-          brew install --build-from-source brew_dist/archivebox.rb
+          pip3 install --upgrade pip setuptools wheel
+          cd brew_dist/
+          brew install --build-bottle ./archivebox.rb
+          # brew bottle archivebox
          archivebox version

-      - name: Upload formula artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox.rb
-          path: brew_dist/archivebox.rb
-
-      - name: Push to homebrew-archivebox tap
-        env:
-          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
+      - name: Add some links to test
        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap
+          mkdir data && cd data
+          archivebox init --setup
+          archivebox add 'https://example.com'
+          archivebox version
+          archivebox status

-          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
-            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb
+      # - name: Commit built package
+      #   run: |
+      #     cd brew_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Homebrew package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/homebrew-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: brew_dist

-          cd /tmp/tap
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add -A
-          if git diff --cached --quiet; then
-            echo "[i] No changes to formula, skipping push."
-          else
-            git commit -m "Update archivebox to v${VERSION}"
-            git push origin HEAD
-            echo "[√] Formula pushed to homebrew-archivebox tap"
-          fi
+      # TODO: push bottle homebrew core PR with latest changes
@@ -0,0 +1,58 @@
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Build GitHub Pages website
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["dev"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Copy README.md into place
+        run: |
+          rm ./website/README.md
+          cp ./README.md ./website/README.md
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2
@@ -5,42 +5,31 @@ on:
  push:

 env:
-  UV_NO_SOURCES: "1"
-  PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
+  MAX_LINE_LENGTH: 110

 jobs:
  lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
        with:
          submodules: true
          fetch-depth: 1

-      - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
-
      - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v1
        with:
-          python-version: "3.13"
+          python-version: 3.9
          architecture: x64

-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: false
-
-      - name: Install dependencies with uv
+      - name: Install flake8
        run: |
-          uv sync --all-extras --all-groups --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          pip install flake8

-      - name: Run prek
-        run: uv run --no-sync --no-sources prek run --all-files
+      - name: Lint with flake8
+        run: |
+          cd archivebox
+          # one pass for show-stopper syntax errors or undefined names
+          flake8 . --count --show-source --statistics
+          # one pass for small stylistic things
+          flake8 . --count --max-line-length="$MAX_LINE_LENGTH" --statistics
@@ -2,64 +2,77 @@ name: Build Pip package

 on:
  workflow_dispatch:
-  workflow_call:
  push:
-    branches:
-      - '**'
-    tags:
-      - 'v*'

-env:
-  PYTHON_VERSION: "3.13"

 jobs:
  build:
+    runs-on: ubuntu-22.04
+
    permissions:
      id-token: write

-    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v4
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
        with:
-          enable-cache: false
+          submodules: true
+          fetch-depth: 1

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: '3.11'
          architecture: x64

-      - name: APT install archivebox dev + run dependencies
-        uses: awalsh128/cache-apt-pkgs-action@latest
+      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

-      - name: UV install archivebox dev + run sub-dependencies
-        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-
-      - name: UV build archivebox and archivebox/pkgs/* packages
-        run: |
-          uv build --all
-
-      - name: Publish new package wheels and sdists to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        # ignore when publish to PyPI fails due to duplicate tag
-        continue-on-error: true
+      - uses: pdm-project/setup-pdm@v3
        with:
-          password: ${{ secrets.PYPI_PAT_SECRET }}
+          python-version: '3.11'
+          cache: true

-      - name: UV install archivebox and archivebox/pkgs/* locally for tests
-        run: uv sync --all-extras --no-sources --no-cache
+      - name: Install dependencies
+        run: pdm install --fail-fast --no-lock --dev --group=':all' --no-self

-      - name: UV run archivebox init + archivebox version
+      - name: Build package
        run: |
-          mkdir -p data && cd data
-          uv run --no-sync --no-sources archivebox init \
-          && uv run --no-sync --no-sources archivebox version
-          # && uv run archivebox add 'https://example.com' \
-          # && uv run archivebox status \
-          # || (echo "UV Failed to run archivebox!" && exit 1)
+          pdm build
+
+      - name: Install from build
+        run: pip install ./dist/archivebox-*.whl
+
+      - name: Add some links to test
+        run: |
+          mkdir data && cd data
+          archivebox init
+          archivebox add 'https://example.com'
+          archivebox version
+          archivebox status
+
+      #- name: Publish package distributions to PyPI
+      #  run: pdm publish --no-build
+
+      # - name: Push build to PyPI
+      #   run: |
+      #     cd pip_dist/
+      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
+      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
+
+      # - name: Commit built package
+      #   run: |
+      #     cd pip_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Pip package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/pip-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: pip_dist
+
@@ -1,45 +0,0 @@
-name: Release State
-
-on:
-  push:
-    branches:
-      - '**'
-  workflow_dispatch:
-
-permissions:
-  contents: write
-  id-token: write
-
-jobs:
-  release-state:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          ref: ${{ github.ref_name }}
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: false
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Configure git identity
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-
-      - name: Run release script
-        env:
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-          GH_TOKEN: ${{ github.token }}
-          PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
-        run: ./bin/release.sh
@@ -1,43 +0,0 @@
-name: Release
-
-# Orchestrates the full release pipeline:
-#   1. Build and publish pip package to PyPI
-#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
-#   3. Build Docker images (in parallel with deb/brew)
-#
-# Individual workflows also run on push for CI (see their own triggers).
-# Manual fallback for the hosted release pipeline. The normal prerelease loop
-# is run by bin/release.sh locally so published GitHub releases do not launch a
-# duplicate release job.
-
-on:
-  workflow_dispatch:
-
-permissions:
-  contents: write
-  packages: write
-  id-token: write
-
-jobs:
-  pip:
-    name: Publish to PyPI
-    uses: ./.github/workflows/pip.yml
-    secrets: inherit
-
-  debian:
-    name: Build .deb packages
-    needs: pip
-    uses: ./.github/workflows/debian.yml
-    secrets: inherit
-
-  homebrew:
-    name: Update Homebrew formula
-    needs: pip
-    uses: ./.github/workflows/homebrew.yml
-    secrets: inherit
-
-  docker:
-    name: Build Docker images
-    needs: pip
-    uses: ./.github/workflows/docker.yml
-    secrets: inherit
@@ -1,291 +0,0 @@
-name: Parallel Tests
-
-on:
-  pull_request:
-    branches: [dev, main, master]
-  push:
-    branches: [dev]
-
-env:
-  PYTHONIOENCODING: utf-8
-  PYTHONLEGACYWINDOWSSTDIO: utf-8
-  USE_COLOR: False
-  UV_NO_SOURCES: "1"
-
-jobs:
-  discover-tests:
-    name: Discover test files
-    runs-on: ubuntu-22.04
-    outputs:
-      test-files: ${{ steps.set-matrix.outputs.test-files }}
-      plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Discover test files
-        id: set-matrix
-        run: |
-          # Find all main test files
-          all_tests=$(find archivebox/tests -maxdepth 1 -name "test_*.py" -type f | sort)
-
-          # Create JSON array with test file info
-          json_array="["
-          first=true
-          for test_file in $all_tests; do
-            if [ "$first" = true ]; then
-              first=false
-            else
-              json_array+=","
-            fi
-
-            # Extract a display name for the test
-            name="main/$(basename $test_file .py | sed 's/^test_//')"
-
-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
-          done
-          json_array+="]"
-
-          echo "test-files=$json_array" >> $GITHUB_OUTPUT
-          echo "Found $(echo $all_tests | wc -w) test files"
-          echo "$json_array" | jq '.'
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Discover plugin tests
-        id: set-plugin-matrix
-        run: |
-          plugin_tests=$(find abx-plugins/abx_plugins/plugins -maxdepth 2 -type d -name tests | sed 's#abx-plugins/abx_plugins/plugins/##; s#/tests##' | sort)
-
-          json_array="["
-          first=true
-          for plugin_name in $plugin_tests; do
-            if [ "$first" = true ]; then
-              first=false
-            else
-              json_array+=","
-            fi
-
-            needs_chromium=false
-            if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
-              needs_chromium=true
-            fi
-
-            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium}"
-          done
-          json_array+="]"
-
-          echo "plugin-tests=$json_array" >> $GITHUB_OUTPUT
-          echo "Found $(echo $plugin_tests | wc -w) plugin test suites"
-          echo "$json_array" | jq '.'
-
-  run-tests:
-    name: ${{ matrix.test.name }}
-    runs-on: ubuntu-22.04
-    needs: discover-tests
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
-      CHROME_HEADLESS: "true"
-      CHROME_SANDBOX: "false"
-      PERSONAS_DIR: /tmp/abx-personas
-      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
-
-    strategy:
-      fail-fast: false
-      matrix:
-        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
-        python: ["3.13"]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
-
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python }}
-          architecture: x64
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          version: "latest"
-
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Cache uv
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-uv-
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.1
-
-      - name: Install dependencies with uv
-        run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
-
-      - name: Run test - ${{ matrix.test.name }}
-        run: |
-          mkdir -p tests/out
-          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
-
-  plugin-tests:
-    name: ${{ matrix.plugin.name }}
-    runs-on: ubuntu-22.04
-    needs: discover-tests
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
-      CHROME_HEADLESS: "true"
-      CHROME_SANDBOX: "false"
-      PERSONAS_DIR: /tmp/abx-personas
-      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
-
-    strategy:
-      fail-fast: false
-      matrix:
-        plugin: ${{ fromJson(needs.discover-tests.outputs.plugin-tests) }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
-
-      - name: Set up Python 3.13
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.13"
-          architecture: x64
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          version: "latest"
-
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Cache uv
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-3.13-uv-
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.1
-
-      - name: Install dependencies with uv
-        run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
-
-      - name: Select Chromium for extension-capable CDP
-        if: ${{ matrix.plugin.needs_chromium }}
-        env:
-          MIN_CHROMIUM_MAJOR: "149"
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR"
-          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-
-          candidate="/usr/bin/chromium"
-          version=""
-          major=0
-          if [ -x "$candidate" ]; then
-            version="$("$candidate" --version || true)"
-            major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-            case "$major" in
-              ''|*[!0-9]*) major=0 ;;
-            esac
-          fi
-
-          if [ "$major" -ge "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Using runner Chromium: $candidate ($version)"
-            echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
-            exit 0
-          fi
-
-          echo "Runner Chromium is unavailable or too old: ${version:-not found}"
-          echo "Installing Playwright Chromium into $LIB_DIR..."
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=npm \
-            --lib="$LIB_DIR" \
-            --postinstall-scripts=false \
-            --min-release-age=0 \
-            --no-cache \
-            --overrides='{"npm":{"install_args":["playwright@next"]}}' \
-            playwright
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=playwright \
-            --lib="$LIB_DIR" \
-            --bin-dir="$LIB_DIR/env/bin" \
-            --min-version="$MIN_CHROMIUM_MAJOR" \
-            chromium
-
-          candidate="$LIB_DIR/env/bin/chromium"
-          if [ ! -x "$candidate" ]; then
-            echo "Playwright Chromium shim was not created at $candidate" >&2
-            exit 1
-          fi
-          version="$("$candidate" --version || true)"
-          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-          case "$major" in
-            ''|*[!0-9]*) major=0 ;;
-          esac
-          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Playwright Chromium is still too old: ${version:-unknown}" >&2
-            exit 1
-          fi
-          echo "Using Playwright Chromium: $candidate ($version)"
-          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
-
-      - name: Run plugin tests - ${{ matrix.plugin.name }}
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
-          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
-        run: |
-          uv run --no-sync --no-sources bash ./bin/test_plugins.sh "${{ matrix.plugin.plugin }}" --no-coverage
@@ -6,19 +6,16 @@ env:
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False
-  UV_NO_SOURCES: "1"

 jobs:
  python_tests:
    runs-on: ${{ matrix.os }}
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl

    strategy:
      matrix:
        os: [ubuntu-22.04]
        # os: [ubuntu-22.04, macos-latest, windows-latest]
-        python: ["3.13"]
+        python: [3.11]

    steps:
      - uses: actions/checkout@v4
@@ -26,15 +23,6 @@ jobs:
          submodules: true
          fetch-depth: 1

-      - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
-
      ### Setup Python & JS Languages
      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@v4
@@ -42,34 +30,63 @@ jobs:
          python-version: ${{ matrix.python }}
          architecture: x64

-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          version: "latest"
-
      - name: Set up Node JS
        uses: actions/setup-node@v4
        with:
-          node-version: 22
+          node-version: 20.10.0
+
+      - name: Setup PDM
+        uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true

      ### Install Python & JS Dependencies
-      - name: Cache uv
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: Cache pip
        uses: actions/cache@v3
+        id: cache-pip
        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-uv-
+            ${{ runner.os }}-${{ matrix.python }}-venv-

      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

-      - name: Install dependencies with uv
+      - name: Install pip dependencies
        run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+          python -m pip install --upgrade pip setuptools wheel pytest bottle build
+          python -m pip install -r requirements.txt
+          python -m pip install -e .[sonic,ldap]
+
+      - name: Get npm cache dir
+        id: npm-cache
+        run: |
+          echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
+
+      - name: Cache npm
+        uses: actions/cache@v3
+        id: cache-npm
+        with:
+          path: ${{ steps.npm-cache.outputs.dir }}
+          key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node_modules
+
+      - name: Install npm requirements
+        run: |
+          npm install
+          echo "SINGLEFILE_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/single-file" >> $GITHUB_ENV
+          echo "READABILITY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor" >> $GITHUB_ENV
+          echo "MERCURY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/mercury-parser" >> $GITHUB_ENV

      ### Run the tests
      - name: Directory listing for debugging
@@ -79,24 +96,13 @@ jobs:

      - name: Archivebox version
        run: |
-          mkdir -p tests/out/data
-          DATA_DIR="$PWD/tests/out/data" uv run --no-sync --no-sources archivebox version
+          archivebox version

      - name: Test built package with pytest
        # TODO: remove this exception for windows once we get tests passing on that platform
        if: ${{ !contains(matrix.os, 'windows') }}
        run: |
-          mkdir -p tests/out
-          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs
-
-      - name: Run plugin tests
-        if: ${{ !contains(matrix.os, 'windows') }}
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
-          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
-        run: |
-          uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage
+          python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist

  docker_tests:
    runs-on: ubuntu-latest
@@ -1,34 +1,22 @@
 .DS_Store
 ._*
-/.heartbeat.json
-/package.json
-/package-lock.json
 *.pyc
 __pycache__/
 .mypy_cache/
 .eggs/
 tests/out/

-# Coverage
-.coverage
-.coverage.*
-coverage.json
-coverage/
-htmlcov/
-
 # Python and Node dependencies
 venv/
 .venv/
 .docker-venv/
 node_modules/
-typings/

 # Ignore dev lockfiles (should always be built fresh)
 pdm.dev.lock
 requirements-dev.txt

 # Packaging artifacts
-requirements.txt
 .pdm-python
 .pdm-build
 archivebox.egg-info
@@ -37,28 +25,12 @@ build/
 dist/

 # Data folders
-lib/
-out/
-tmp/
 data/
 data*/
-archivebox/tests/data/
-archive/
 output/
-logs/
-/-
-/personas/
-/sources/
 index.sqlite3
-queue.sqlite3
 *.sqlite*
 data.*
-.archivebox_id
-ArchiveBox.conf
-*.stdout
-*.stderr
-*.log
-.tmp/

 # vim
 *.sw?
@@ -0,0 +1,31 @@
+[submodule "docs"]
+    path = docs
+    url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
+
+[submodule "deb_dist"]
+    path = deb_dist
+    url = https://github.com/ArchiveBox/debian-archivebox.git
+[submodule "brew_dist"]
+	path = brew_dist
+	url = https://github.com/ArchiveBox/homebrew-archivebox.git
+[submodule "pip_dist"]
+	path = pip_dist
+	url = https://github.com/ArchiveBox/pip-archivebox.git
+[submodule "docker"]
+	path = docker
+	url = https://github.com/ArchiveBox/docker-archivebox.git
+[submodule "archivebox/vendor/base32-crockford"]
+	path = archivebox/vendor/base32-crockford
+	url = https://github.com/jbittel/base32-crockford
+[submodule "archivebox/vendor/pocket"]
+	path = archivebox/vendor/pocket
+	url = https://github.com/tapanpandita/pocket
+[submodule "archivebox/vendor/django-taggit"]
+	path = archivebox/vendor/django-taggit
+	url = https://github.com/jazzband/django-taggit
+[submodule "archivebox/vendor/python-atomicwrites"]
+	path = archivebox/vendor/python-atomicwrites
+	url = https://github.com/untitaker/python-atomicwrites
+[submodule "archivebox/vendor/pydantic-pkgr"]
+	path = archivebox/vendor/pydantic-pkgr
+	url = https://github.com/ArchiveBox/pydantic-pkgr
@@ -0,0 +1,19 @@
+tests/
+archivebox/
+archivebox.egg-info/
+build/
+dist/
+docs/
+etc/
+.github
+.gitmodules
+.dockerignore
+.flake8
+CNAME
+_config.yml
+docker-compose.yaml
+docker-compose.yml
+Dockerfile
+MANIFEST.in
+Pipfile
+setup.py
@@ -1,67 +0,0 @@
-default_language_version:
-  python: python3.13
-
-repos:
-  - repo: https://github.com/asottile/yesqa
-    rev: v1.5.0
-    hooks:
-      - id: yesqa
-
-  - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.1
-    hooks:
-      - id: codespell
-        additional_dependencies:
-          - tomli
-
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.20.0
-    hooks:
-      - id: pyupgrade
-        args: [--py313-plus]
-
-  - repo: https://github.com/asottile/add-trailing-comma
-    rev: v3.1.0
-    hooks:
-      - id: add-trailing-comma
-
-  - repo: local
-    hooks:
-      - id: ruff-format
-        name: ruff-format
-        entry: uv run --active --no-sources ruff format
-        language: system
-        types_or: [python, pyi]
-      - id: ruff-check
-        name: ruff-check
-        entry: uv run --active --no-sources ruff check --fix
-        language: system
-        types_or: [python, pyi]
-
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
-    hooks:
-      - id: check-ast
-      - id: check-toml
-      - id: check-yaml
-        exclude: ^\.github/workflows/homebrew\.yml$
-      - id: check-json
-      - id: check-merge-conflict
-      - id: check-symlinks
-      - id: destroyed-symlinks
-      - id: check-case-conflict
-      - id: check-illegal-windows-names
-      - id: check-shebang-scripts-are-executable
-        exclude: ^(archivebox/.*\.py|archivebox/tests/.*\.py|archivebox/personas/export_browser_state\.js)$
-      - id: mixed-line-ending
-      - id: fix-byte-order-marker
-      - id: end-of-file-fixer
-      - id: detect-private-key
-      - id: debug-statements
-      - id: forbid-submodules
-        exclude: ^docs$
-      - id: check-added-large-files
-        args: ["--maxkb=600"]
-      - id: name-tests-test
-        args: ["--pytest-test-first"]
-        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)
@@ -0,0 +1,33 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+submodules:
+  include: all
+  recursive: true
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    nodejs: "20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+  - pdf
+  - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt
@@ -1,497 +0,0 @@
-# Claude Code Development Guide for ArchiveBox
-
-## Quick Start
-
-```bash
-# Set up dev environment (always use uv, never pip directly)
-uv sync --dev --all-extras
-
-# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
-```
-
-## Development Environment Setup
-
-### Prerequisites
- Python 3.11+ (3.13 recommended)
- uv package manager
- A non-root user for running tests (e.g., `testuser`)
-
-### Install Dependencies
-```bash
-uv sync --dev --all-extras  # Always use uv, never pip directly
-```
-
-### Activate Virtual Environment
-```bash
-source .venv/bin/activate
-```
-
-### Common Gotchas
-
-#### File Permissions
-New files created by root need permissions fixed for testuser:
-```bash
-chmod 644 archivebox/tests/test_*.py
-```
-
-#### DATA_DIR Environment Variable
-ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
-
-## Code Style Guidelines
-
-### Naming Conventions for Grep-ability
-Use consistent naming for everything to enable easy grep-ability and logical grouping:
-
-**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
-
-**Examples**:
-```python
-# Filesystem migration methods - all start with fs_
-def fs_migration_needed() -> bool: ...
-def fs_migrate() -> None: ...
-def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
-def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
-def _fs_next_version(current: str) -> str: ...
-
-# Logging methods - ALL must start with log_ or _log
-def log_migration_start(snapshot_id: str) -> None: ...
-def _log_error(message: str) -> None: ...
-def log_validation_result(ok: bool, msg: str) -> None: ...
-```
-
-**Rules**:
- Group related functions with common prefixes
- Use `_` prefix for internal/private helpers within the same family
- ALL logging-related methods MUST start with `log_` or `_log`
- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
-
-### Minimize Unique Names and Data Structures
-**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
-
-**Example - GOOD**:
-```python
-# Binary has overrides field
-binary = Binary(overrides={'TIMEOUT': '60s'})
-
-# Binary reuses the same field name and structure
-class Binary(models.Model):
-    overrides = models.JSONField(default=dict)  # Same name, same structure
-```
-
-**Example - BAD**:
-```python
-# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
-class Binary(models.Model):
-    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
-```
-
-**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
-
-## Testing
-
-### CRITICAL: Never Run as Root
-ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
-
-```bash
-# Run all migration tests
-sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
-
-# Run specific test file
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
-
-# Run single test
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
-```
-
-### Test File Structure
-```
-archivebox/tests/
-├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
-├── test_migrations_fresh.py      # Fresh install tests
-├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
-├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
-└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
-```
-
-### Test Writing Standards
-
-#### NO MOCKS - Real Tests Only
-Tests must exercise real code paths:
- Create real SQLite databases with version-specific schemas
- Seed with realistic test data
- Run actual `python -m archivebox` commands via subprocess
- Query SQLite directly to verify results
-
-**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
-
-#### NO SKIPS
-Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
-
-#### Strict Assertions
- `init` command must return exit code 0 (not `[0, 1]`)
- Verify ALL data is preserved, not just "at least one"
- Use exact counts (`==`) not loose bounds (`>=`)
-
-### Example Test Pattern
-```python
-def test_migration_preserves_snapshots(self):
-    """Migration should preserve all snapshots."""
-    result = run_archivebox(self.work_dir, ['init'], timeout=45)
-    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
-
-    ok, msg = verify_snapshot_count(self.db_path, expected_count)
-    self.assertTrue(ok, msg)
-```
-
-### Testing Gotchas
-
-#### Extractors Disabled for Speed
-Tests disable all extractors via environment variables for faster execution:
-```python
-env['SAVE_TITLE'] = 'False'
-env['SAVE_FAVICON'] = 'False'
-# ... etc
-```
-
-#### Timeout Settings
-Use appropriate timeouts for migration tests (45s for init, 60s default).
-
-### Plugin Testing & Code Coverage
-
-**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
-
-```bash
-# Run plugin tests with coverage (both Python + JavaScript)
-bash bin/test_plugins.sh screenshot
-
-# View coverage reports
-bash bin/test_plugins.sh --coverage-report
-# Or individual reports:
-coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
-```
-
-#### Plugin Test Structure
-
-Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
-
-```python
-# Correct production paths:
-# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
-# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
-
-with tempfile.TemporaryDirectory() as tmpdir:
-    data_dir = Path(tmpdir)
-
-    # Crawl-level plugin (e.g., chrome launcher)
-    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
-    chrome_dir = crawl_dir / 'chrome'
-    chrome_dir.mkdir(parents=True)
-
-    # Snapshot-level plugin (e.g., screenshot)
-    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
-    screenshot_dir = snapshot_dir / 'screenshot'
-    screenshot_dir.mkdir(parents=True)
-
-    # Run hook in its output directory
-    result = subprocess.run(
-        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
-        cwd=str(screenshot_dir),
-        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
-        capture_output=True,
-        timeout=120
-    )
-```
-
-#### Coverage Improvement Loop
-
-To improve from ~20% to 80%+:
-
-1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
-2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
-3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
-4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
-
-**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
-
-## Database Migrations
-
-### Generate and Apply Migrations
-```bash
-# Generate migrations (run from archivebox subdirectory)
-cd archivebox
-./manage.py makemigrations
-
-# Apply migrations to test database
-cd data/
-archivebox init
-```
-
-### Schema Versions
- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
-
-### Testing a Migration Path
-1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
-2. Seed with realistic test data using `seed_0_X_data()`
-3. Run `archivebox init` to trigger migrations
-4. Verify data preservation with `verify_*` functions
-5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
-
-### Squashed Migrations
-When testing 0.8.x (dev branch), you must record ALL replaced migrations:
-```python
-# The squashed migration replaces these - all must be recorded
-('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
-('core', '0024_auto_20240513_1143'),
-# ... all 52 migrations from 0023-0074 ...
-('core', '0023_new_schema'),  # Also record the squashed migration itself
-```
-
-### Migration Strategy
- Squashed migrations for clean installs
- Individual migrations recorded for upgrades from dev branch
- `replaces` attribute in squashed migrations lists what they replace
-
-### Migration Gotchas
-
-#### Circular FK References in Schemas
-SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
-
-## Plugin System Architecture
-
-### Plugin Dependency Rules
-
-Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
-However, they are allowed to depend on two shared files ONLY:
- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
-
-### Chrome-Dependent Plugins
-
-Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
-
-**Main puppeteer-based chrome installer + launcher plugin**:
- `chrome` - Core Chrome integration (CDP, launch, navigation)
-
-**Metadata extraction using chrome/chrome_utils.js / CDP**:
- `dns` - DNS resolution info
- `ssl` - SSL certificate info
- `headers` - HTTP response headers
- `redirects` - Capture redirect chains
- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
- `responses` - Capture network responses
- `consolelog` - Capture console.log output
- `title` - Extract page title
- `accessibility` - Extract accessibility tree
- `seo` - Extract SEO metadata
-
-**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
- `ublock` - uBlock Origin ad blocking
- `istilldontcareaboutcookies` - Cookie banner dismissal
- `twocaptcha` - 2captcha CAPTCHA solver integration
-
-**Page-alteration plugins to prepare the content for archiving**:
- `modalcloser` - Modal dialog dismissal
- `infiniscroll` - Infinite scroll handler
-
-**Main Extractor Outputs**:
- `dom` - DOM snapshot extraction
- `pdf` - Generate PDF snapshots
- `screenshot` - Generate screenshots
- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
-
-**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
-
-### Finding Chrome-Dependent Plugins
-
-```bash
-# Find all files containing "chrom" (case-insensitive)
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
-
-# Or get just the plugin names
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
-```
-
-**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
-
-## Architecture Notes
-
-### Crawl Model (0.9.x)
- Crawl groups multiple Snapshots from a single `add` command
- Each `add` creates one Crawl with one or more Snapshots
- Seed model was removed - crawls now store URLs directly
-
-## Code Coverage
-
-### Overview
-
-Coverage tracking is enabled for passive collection across all contexts:
- Unit tests (pytest)
- Integration tests
- Dev server (manual testing)
- CLI usage
-
-Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
-
-### Install Coverage Tools
-
-```bash
-uv sync --dev  # Installs pytest-cov and coverage
-```
-
-### Running with Coverage
-
-#### Unit Tests
-```bash
-# Run tests with coverage
-pytest --cov=archivebox --cov-report=term archivebox/tests/
-
-# Or run specific test file
-pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
-```
-
-#### Dev Server with Coverage
-```bash
-# Start dev server with coverage tracking
-coverage run --parallel-mode -m archivebox server
-
-# Or CLI commands
-coverage run --parallel-mode -m archivebox init
-coverage run --parallel-mode -m archivebox add https://example.com
-```
-
-#### Manual Testing (Always-On)
-To enable coverage during ALL Python executions (passive tracking):
-
-```bash
-# Option 1: Use coverage run wrapper
-coverage run --parallel-mode -m archivebox [command]
-
-# Option 2: Set environment variable (tracks everything)
-export COVERAGE_PROCESS_START=pyproject.toml
-# Now all Python processes will track coverage
-archivebox server
-archivebox add https://example.com
-```
-
-### Viewing Coverage
-
-#### Text Report (Quick View)
-```bash
-# Combine all parallel coverage data
-coverage combine
-
-# View summary
-coverage report
-
-# View detailed report with missing lines
-coverage report --show-missing
-
-# View specific file
-coverage report --include="archivebox/core/models.py" --show-missing
-```
-
-#### JSON Report (LLM-Friendly)
-```bash
-# Generate JSON report
-coverage json
-
-# View the JSON
-cat coverage.json | jq '.files | keys'  # List all files
-
-# Find files with low coverage
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
-
-# Find completely uncovered files (dead code candidates)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# Get missing lines for a specific file
-cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
-```
-
-#### HTML Report (Visual)
-```bash
-# Generate interactive HTML report
-coverage html
-
-# Open in browser
-open htmlcov/index.html
-```
-
-### Isolated Runs
-
-To measure coverage for specific scenarios:
-
-```bash
-# 1. Reset coverage data
-coverage erase
-
-# 2. Run your isolated test/scenario
-pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
-# OR
-coverage run --parallel-mode -m archivebox add https://example.com
-
-# 3. View results
-coverage combine
-coverage report --show-missing
-
-# 4. Optionally export for analysis
-coverage json
-```
-
-### Finding Dead Code
-
-```bash
-# 1. Run comprehensive tests + manual testing to build coverage
-pytest --cov=archivebox archivebox/tests/
-coverage run --parallel-mode -m archivebox server  # Use the app manually
-coverage combine
-
-# 2. Find files with 0% coverage (strong dead code candidates)
-coverage json
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# 3. Find files with <10% coverage (likely dead code)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
-
-# 4. Generate detailed report for analysis
-coverage report --show-missing > coverage_report.txt
-```
-
-### Tips
-
- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
- **Reset**: Use `coverage erase` to start fresh for isolated measurements
- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
-
-## Debugging Tips
-
-### Check Migration State
-```bash
-sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
-```
-
-### Check Table Schema
-```bash
-sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
-```
-
-### Verbose Test Output
-```bash
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
-```
-
-### Kill Zombie Chrome Processes
-```bash
-./bin/kill_chrome.sh
-```
@@ -1,38 +1,22 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
-#     python3.13, uv, python3-ldap
-#     curl, wget, git, dig, ping, tree, nano
-#     node, npm
-#     ArchiveBox and plugin runtime dependencies installed by archivebox init --install
+# This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
+#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file
 # Usage:
-#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
-#     docker build . -t archivebox
+#     git submodule update --init --recursive
+#     git pull --recurse-submodules
+#     docker build . -t archivebox --no-cache
 #     docker run -v "$PWD/data":/data archivebox init
 #     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
 #     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
-#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
-# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+# Multi-arch build:
+#     docker buildx create --use
+#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
+#
+# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).


-#########################################################################################
-
-### Example: Using ArchiveBox in your own project's Dockerfile ########
-
-# FROM python:3.13-slim
-# WORKDIR /data
-# RUN pip install archivebox>=0.9.0   # use latest release here
-# RUN archivebox install
-# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
-
-#########################################################################################
-
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT=
-
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04
+# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
+FROM python:3.11-slim-bookworm

 LABEL name="archivebox" \
    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -58,9 +42,10 @@ ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
+
 ######### Environment Variables #################################

-# Global build-time and runtime environment constants + default pkg manager config
+# Global system-level config
 ENV TZ=UTC \
    LANGUAGE=en_US:en \
    LC_ALL=C.UTF-8 \
@@ -72,51 +57,54 @@ ENV TZ=UTC \
    PIP_DISABLE_PIP_VERSION_CHECK=1 \
    npm_config_loglevel=error

-# Language Version config
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
+# Version config
+ENV PYTHON_VERSION=3.11 \
+    NODE_VERSION=20

-# Non-root User config
+# User config
 ENV ARCHIVEBOX_USER="archivebox" \
    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
+    DEFAULT_PGID=911

-# ArchiveBox Source Code + Lib + Data paths
+# Global paths
 ENV CODE_DIR=/app \
    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    LIB_BIN_DIR=/opt/archivebox/lib/bin \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    GLOBAL_VENV=/venv \
    PLAYWRIGHT_BROWSERS_PATH=/browsers

-# Bash SHELL config
-# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+# Application-level paths
+ENV APP_VENV=/app/.venv \
+    NODE_MODULES=/app/node_modules
+
+# Build shell config
+ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 

 ######### System Environment ####################################

-# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
-WORKDIR "$CODE_DIR"
+# Detect ArchiveBox version number by reading package.json
+COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
+RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt

-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
+# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
 RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
    && rm -f /etc/apt/apt.conf.d/docker-clean

 # Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox starting..." \
+RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
    && echo \
+    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
    && echo \
    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which bash && bash --version | sed -n '1p' \
-    && which dpkg && dpkg --version | sed -n '1p' \
+    && cat /etc/os-release | head -n7 \
+    && which bash && bash --version | head -n1 \
+    && which dpkg && dpkg --version | head -n1 \
    && echo -e '\n\n' && env && echo -e '\n\n' \
    ) | tee -a /VERSION.txt

@@ -128,43 +116,35 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
+    # DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
    # https://docs.linuxserver.io/general/understanding-puid-and-pgid

 # Install system apt dependencies (adding backports to access more recent apt updates)
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
+    echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
+    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' > /etc/apt/sources.list.d/backports.list \
    && mkdir -p /etc/apt/keyrings \
    && apt-get update -qq \
-    && apt-get install -qq -y \
+    && apt-get install -qq -y -t bookworm-backports \
        # 1. packaging dependencies
        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils git python3.12-venv \
+        zlib1g-dev dumb-init gosu cron unzip grep \
        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        tree nano iputils-ping \
        # nano iputils-ping dnsutils htop procps jq yq
    && rm -rf /var/lib/apt/lists/*

-# Install sonic search backend
-COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-RUN (which sonic && sonic --version) | tee -a /VERSION.txt
-
 ######### Language Environments ####################################

-# Set up Python environment
-# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
-#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
+# Install Python environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
    # && apt-get update -qq \
-    # && apt-get install -qq -y --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
+    # && apt-get install -qq -y -t bookworm-backports --no-upgrade \
+    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip \
    # && rm -rf /var/lib/apt/lists/* \
    # tell PDM to allow using global system python site packages
    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
@@ -172,30 +152,26 @@ RUN (which sonic && sonic --version) | tee -a /VERSION.txt
    # install global dependencies / python build dependencies in GLOBAL_VENV
    # && pip install --upgrade pip setuptools wheel \
    # Save version info
-    # && ( \
-    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
-    #     && which pip && pip --version \
-    #     # && which pdm && pdm --version \
-    #     && echo -e '\n\n' \
-    # ) | tee -a /VERSION.txt
+    && ( \
+        which python3 && python3 --version | grep " $PYTHON_VERSION" \
+        && which pip && pip --version \
+        # && which pdm && pdm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt


-# Set up Node environment from the official platform tarball. This avoids
-# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
-# emulated Docker builds.
-RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    case "$TARGETARCH" in \
-        amd64) NODE_DIST_ARCH="x64" ;; \
-        arm64) NODE_DIST_ARCH="arm64" ;; \
-        *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \
-    esac \
-    && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \
-    && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \
-    && grep "  ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \
-    && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \
-    && rm "$NODE_TARBALL" SHASUMS256.txt \
+# Install Node environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
+    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-upgrade libatomic1 \
+    && apt-get install -y -t bookworm-backports --no-upgrade \
+        nodejs \
+    && rm -rf /var/lib/apt/lists/* \
+    # Update NPM to latest version
+    && npm i -g npm --cache /root/.npm \
    # Save version info
    && ( \
        which node && node --version \
@@ -204,179 +180,147 @@ RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGE
    ) | tee -a /VERSION.txt


-# Set up uv and main app /venv
-RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
-ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv
-WORKDIR "$CODE_DIR"
-# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
-    && uv venv /venv --python ${PYTHON_VERSION}
-ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
-RUN uv pip install setuptools pip \
+######### Extractor Dependencies ##################################
+
+# Install apt dependencies
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing APT extractor dependencies globally using apt..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports \
+        curl wget git yt-dlp ffmpeg ripgrep \
+        # Packages we have also needed in the past:
+        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
+    && rm -rf /var/lib/apt/lists/* \
+    # Save version info
    && ( \
-        which python3 && python3 --version \
-        && which uv && uv self version \
-        && uv python find --system && uv python find \
+        which curl && curl --version | head -n1 \
+        && which wget && wget --version 2>&1 | head -n1 \
+        && which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
+        && which git && git --version 2>&1 | head -n1 \
+        && which rg && rg --version 2>&1 | head -n1 \
        && echo -e '\n\n' \
    ) | tee -a /VERSION.txt

-
-######### ArchiveBox & Extractor Dependencies ##################################
-
-# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
+# Install chromium browser using playwright
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc \
-        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
-        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    && uv pip install \
-        "python-ldap>=3.4.3" \
-    && apt-get purge -y \
-        python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
+    && apt-get install -qq -y -t bookworm-backports \
+        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+        at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
+        libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
+        libxaw7 libxcomposite1 libxdamage1 libxfont2 \
+        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils xfonts-encodings \
+        # xfonts-scalable xfonts-utils xserver-common xvfb \
+        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
+        # libxss1 dbus dbus-x11 upower \
+    # && service dbus start \
+    # install Chromium using playwright
+    && pip install playwright \
+    && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
+    && playwright install chromium \
+    && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
+    && rm -rf /var/lib/apt/lists/* \
+    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
+    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
+    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
+    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
+    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
+    # Save version info
+    && ( \
+        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt

-
-# Runtime config used by plugin hooks. Plugin binaries and npm packages are
-# installed into LIB_DIR below by archivebox init --install and resolved from
-# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
-ENV PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+# Install Node dependencies
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
+RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
+    && npm ci --prefer-offline --no-audit --cache /root/.npm \
+    && ( \
+        which node && node --version \
+        && which npm && npm version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt

 ######### Build Dependencies ####################################

-
-# Install ArchiveBox Python venv dependencies from pyproject.toml.
-RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \
-    && uv --no-cache sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/*
-    # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies
-
-# Setup ArchiveBox runtime config
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/usr/bin/python3.12 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-WORKDIR "$DATA_DIR"
-RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "$DATA_DIR" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \
-    && mkdir -p "$TMP_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
-    && mkdir -p "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer
-# path users run later, before copying ArchiveBox source so source-only edits do
-# not invalidate the heavy browser/plugin dependency layer.
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
-    && apt-get update -qq \
-    && if [ "$TARGETARCH" = "arm64" ]; then \
-        abxpkg install --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
-    fi \
-    && PUID=0 PGID=0 abx-dl plugins --install \
-    && find "$LIB_DIR" "$DATA_DIR"/personas -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$LIB_DIR" "$DATA_DIR"/personas -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/* \
-    && (chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR"/personas 2>/dev/null || true) \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
-
-# Install ArchiveBox Python package from the checked-out source.
+# Install ArchiveBox Python dependencies
 WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports \
+        build-essential \
+        libssl-dev libldap2-dev libsasl2-dev \
+        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
+    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
+    # && pdm use --venv in-project \
+    # && pdm run python -m ensurepip \
+    # && pdm sync --fail-fast --no-editable --group :all --no-self \
+    # && pdm export -o requirements.txt --without-hashes \
+    # && source $GLOBAL_VENV/bin/activate \
+    && pip install -r requirements.txt \
+    && apt-get purge -y \
+        build-essential \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install ArchiveBox Python package from source
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && pip install \
-        --no-deps \
-        "$CODE_DIR" \
-    && ( \
-        pip show archivebox \
-        && which archivebox \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
-    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
-
-# Initialize an empty image collection without rerunning dependency installs.
-WORKDIR "$DATA_DIR"
-RUN echo "[+] Initializing image collection..." \
-    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
-        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
-        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
-        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
-        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        2>/dev/null || true)
-
-# Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
-    ) | tee -a /VERSION.txt
-
-# Verify ArchiveBox is installed and write full version/dependency info.
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
-    && gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
+    # && apt-get update -qq \
+    # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
+    # && apt-get install -qq -y -t bookworm-backports \
+        # build-essential  \
+    # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
+    && pip install -e "$CODE_DIR"[sonic,ldap] \
+    # save docker image size and always remove compilers / build tools after building is complete
+    # && apt-get purge -y build-essential \
+    # && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*

 ####################################################

-# Expose ArchiveBox's main interfaces to the outside world
+# Setup ArchiveBox runtime config
+WORKDIR "$DATA_DIR"
+ENV IN_DOCKER=True \
+    DISPLAY=novnc:0.0 \
+    CUSTOM_TEMPLATES_DIR=/data/templates \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no \
+    ALLOWED_HOSTS=*
+    ## No need to set explicitly, these values will be autodetected by archivebox in docker:
+    # WGET_BINARY="wget" \
+    # YOUTUBEDL_BINARY="yt-dlp" \
+    # CHROME_BINARY="/usr/bin/chromium-browser" \
+    # USE_SINGLEFILE=True \
+    # SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
+    # USE_READABILITY=True \
+    # READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
+    # USE_MERCURY=True \
+    # MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"
+
+# Print version for nice docker finish summary
+RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
+    ) | tee -a /VERSION.txt
+RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
+
+####################################################
+
+# Open up the interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
+    CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'

 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]
+CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]
@@ -1,6 +1,6 @@
 MIT License

-Copyright (c) 2024 Nick Sweeting
+Copyright (c) 2020 Nick Sweeting

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -13,25 +13,23 @@
 <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE"><img src="https://img.shields.io/badge/Open_source-MIT-green.svg?logo=git&logoColor=green"/></a> <a href="https://github.com/ArchiveBox/ArchiveBox/commits/dev"><img src="https://img.shields.io/github/last-commit/ArchiveBox/ArchiveBox.svg?logo=Sublime+Text&logoColor=green&label=Active"/></a> &nbsp; <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?logo=github&label=Stars&logoColor=blue"/></a> &nbsp; <a href="https://hub.docker.com/r/archivebox/archivebox"><img src="https://img.shields.io/docker/pulls/archivebox/archivebox.svg?label=Docker+Pulls"/></a> <a href="https://pypi.org/project/archivebox/"><img src="https://img.shields.io/pypi/dm/archivebox?label=PyPI%20Installs&color=%235f7dae"/></a> <a href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj"><img src="https://img.shields.io/chrome-web-store/users/habonpimjphpdnmcfkaockjnffodikoj?label=Chrome%20Store&color=%231973e8"/></a>

 <!--<pre lang="bash" align="left"><code style="white-space: pre-line; text-align: left" align="left">
-curl -fsSL 'https://get.archivebox.io' | bash    # (or see pip/brew/Docker instructions below)
+curl -fsSL 'https://get.archivebox.io' | sh    # (or see pip/brew/Docker instructions below)
 </code></pre>-->

 </div>
 <hr/>
 <br/>

-**ArchiveBox is a self-hosted app that lets you preserve content from websites in a variety of formats.**
+**ArchiveBox is a powerful, self-hosted internet archiving solution to collect, save, and view websites offline.**

-We aim to make your data immediately useful, and kept in formats that other programs can read directly. As output, we save standard HTML, PNG, PDF, TXT, JSON, WARC, SQLite, all guaranteed to be readable for decades to come. ArchiveBox also has a CLI, REST API, and webhooks so you can set up integrations with other services.
-
-Without active preservation effort, everything on the internet eventually disappears or degrades.
+Without active preservation effort, everything on the internet eventually dissapears or degrades. Archive.org does a great job as a centralized service, but saved URLs have to be public, and they can't save every type of content.

 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>

-> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
+> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart), [macOS](#quickstart), and [Windows](#quickstart) (WSL2), or via **[Docker](#quickstart)** ⭐️.  

-*Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*
+*Once installed, it can be used as a [CLI tool](#usage), [self-hosted Web App](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python library](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [one-off command](#static-archive-exporting).*

 <br/>
 <hr/>
@@ -77,7 +75,7 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 <pre lang="bash"><code style="white-space: pre-line"># Option A: Get ArchiveBox with Docker Compose (recommended):
 mkdir -p ~/archivebox/data && cd ~/archivebox
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml   # edit options in this file as-needed
-docker compose run archivebox init
+docker compose run archivebox init --setup
 # docker compose run archivebox add 'https://example.com'
 # docker compose run archivebox help
 # docker compose up
@@ -85,7 +83,7 @@ docker compose run archivebox init
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -it -v $PWD:/data archivebox/archivebox init
+docker run -it -v $PWD:/data archivebox/archivebox init --setup
 # docker run -it -v $PWD:/data archivebox/archivebox add 'https://example.com'
 # docker run -it -v $PWD:/data archivebox/archivebox help
 # docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox
@@ -94,19 +92,17 @@ docker run -it -v $PWD:/data archivebox/archivebox init
 # Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
 pip install archivebox
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init
-archivebox install
+archivebox init --setup
 # archivebox add 'https://example.com'
 # archivebox help
 # archivebox server 0.0.0.0:8000
 <br/>
 <br/>
 # Option D: Or use the optional auto setup script to install it
-curl -fsSL 'https://get.archivebox.io' | bash
+curl -fsSL 'https://get.archivebox.io' | sh
 </code></pre>
 <br/>
-<sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
-<sub>Set <code>LISTEN_HOST</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
+<sub>Open <a href="http://localhost:8000"><code>http://localhost:8000</code></a> to see your server's Web UI ➡️</sub>
 </details>
 <br/>

@@ -134,7 +130,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
+- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
 - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
 - Advanced users: support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
@@ -154,9 +150,9 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 - **Individuals:**
  `saving bookmarks`, `preserving portfolio content`, `legacy / memoirs archival`
 - **Governments:**
-  `snapshotting public service sites`, `recordkeeping compliance`
+  `snapshoting public service sites`, `recordkeeping compliance`

-> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  
+> ***[Contact us](https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102)** if your org wants help using ArchiveBox professionally.*  (we are also seeking [grant funding](https://github.com/ArchiveBox/ArchiveBox/issues/1126#issuecomment-1487431394))  
 > We offer: setup & support, CAPTCHA/ratelimit unblocking, SSO, audit logging/chain-of-custody, and more  
 > *ArchiveBox is a 🏛️ 501(c)(3) [nonprofit FSP](https://hackclub.com/hcb/) and all our work supports open-source development.* 

@@ -190,9 +186,9 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
 </code></pre></li>
 <li>Run the initial setup to create an admin user (or set ADMIN_USER/PASS in docker-compose.yml)
-<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init
+<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init --setup
 </code></pre></li>
-<li>Next steps: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
+<li>Next steps: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">docker compose up
 # completely optional, CLI can always be used without running a server
 # docker compose run [-T] archivebox [subcommand] [--help]
@@ -214,10 +210,10 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -v $PWD:/data -it archivebox/archivebox init
+docker run -v $PWD:/data -it archivebox/archivebox init --setup
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox
 # completely optional, CLI can always be used without running a server
 # docker run -v $PWD:/data -it [subcommand] [--help]
@@ -237,7 +233,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (optional, highly recommended but not required).</li>
 <li>Run the automatic setup script.
-<pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
+<pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | sh</code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>
@@ -260,22 +256,23 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 <br/>
 <ol>

-<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.13</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v22</a> on your system (if not already installed).</li>
-<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://docs.astral.sh/uv/guides/tools/#running-tools"><code>uvx</code></a>).
-<pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox
+<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.10</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v18</a> on your system (if not already installed).</li>
+<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://pipx.pypa.io"><code>pipx</code></a>).
+<pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox yt-dlp playwright
+playwright install --with-deps chromium
 archivebox version
 # install any missing extras shown using apt/brew/pkg/etc. see Wiki for instructions
-#    python@3.13 node curl wget git ripgrep ...
+#    python@3.10 node curl wget git ripgrep ...
 </code></pre>
 <i>See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install">Install: Bare Metal</a> Wiki for full install instructions for each OS...</i>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data   # for example
-archivebox init     # instantialize a new collection
-archivebox install  # install all the runtime dependencies (e.g. chrome, single-file, yt-dlp, etc.)
+archivebox init --setup   # instantialize a new collection
+# (--setup auto-installs and link JS dependencies: singlefile, readability, mercury, etc.)
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
@@ -294,23 +291,29 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <details>
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
-<ol>
-<li>Download and install the <code>.deb</code> package from the <a href="https://github.com/ArchiveBox/ArchiveBox/releases">latest release</a>.
-<pre lang="bash"><code style="white-space: pre-line"># download the .deb for your architecture (amd64 or arm64)
-ARCH="$(dpkg --print-architecture)"
-VERSION="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")"
-curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${VERSION}_${ARCH}.deb" -o /tmp/archivebox.deb
-sudo apt install /tmp/archivebox.deb
+See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki for instructions. ➡️
+<!--<ol>
+<li>Add the ArchiveBox repository to your sources.<br/>
+<pre lang="bash"><code style="white-space: pre-line">echo "deb http://ppa.launchpad.net/archivebox/archivebox/ubuntu focal main" | sudo tee /etc/apt/sources.list.d/archivebox.list
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C258F79DCC02E369
+sudo apt update
+</code></pre>
+</li>
+<li>Install the ArchiveBox package using <code>apt</code>.
+<pre lang="bash"><code style="white-space: pre-line">sudo apt install archivebox
+# update to newest version with pip (sometimes apt package is outdated)
+pip install --upgrade --ignore-installed archivebox yt-dlp playwright
+playwright install --with-deps chromium    # install chromium and its system dependencies
 archivebox version                         # make sure all dependencies are installed
 </code></pre>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --install
+archivebox init --setup
 </code></pre>
 <br/>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
@@ -319,7 +322,7 @@ archivebox help
 </li>
 </ol>
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
-<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>
+<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>-->
 <br/><br/>
 </details>

@@ -331,17 +334,19 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install the ArchiveBox package using <code>brew</code>.
 <pre lang="bash"><code style="white-space: pre-line">brew tap archivebox/archivebox
 brew install archivebox
+# update to newest version with pip (sometimes brew package is outdated)
+pip install --upgrade --ignore-installed archivebox yt-dlp playwright
+playwright install --with-deps chromium    # install chromium and its system dependencies
 archivebox version                         # make sure all dependencies are installed
 </code></pre>
 <i>See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki for more granular instructions for macOS... ➡️</i>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init
-archivebox install
+archivebox init --setup
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
+<li>Optional: Start the server then login to the Web UI <a href="http://127.0.0.1:8000">http://127.0.0.1:8000</a> ⇢ Admin.
 <pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
@@ -362,8 +367,8 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 > *Warning: These are contributed by external volunteers and may lag behind the official `pip` channel.*

 <ul>
-<li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>, maintained by <a href="https://github.com/jasongodev"><code>@jasongodev</code></a>)</li>
-<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
+<li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>)</li>
+<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | sh</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
 <li>Nix: <a href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/archivebox/default.nix"><code>nix-env --install archivebox</code></a> (contributed by <a href="https://github.com/siraben"><code>@siraben</code></a>)</li>
 <li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
 <li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
@@ -404,7 +409,6 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <ul>
 <li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
 <li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
-<li><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a></li>
 <li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
 <li><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a></li>
 <li><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a></li>
@@ -425,7 +429,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <ul>
 <li><a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">
 <img src="https://img.shields.io/badge/Custom_Development-ArchiveBox.io-%231a1a1a.svg?style=flat" height="22px"/>
-</a> (<a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">get hosting, support, and feature customization directly from us</a>)</li>
+</a> (<a href="https://zulip.archivebox.io/#narrow/stream/167-enterprise/topic/welcome/near/1191102">get hosting, support, and feature customization directy from us</a>)</li>
 <li><a href="https://monadical.com">
 <img src="https://img.shields.io/badge/General_Dev_Consulting-Monadical.com-%231a1a1a.svg?style=flat" height="22px"/>
 </a> (<a href="https://monadical.com/contact-us.html">generalist consultancy that has ArchiveBox experience</a>)</li>
@@ -462,7 +466,6 @@ For more discussion on managed and paid hosting options see here: <a href="https
 #### ➡️&nbsp; Next Steps

 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
- (Optional) Create a persona and import browser cookies to archive logged-in sites: `archivebox persona create --import=chrome personal`
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -489,19 +492,13 @@ docker compose run archivebox help

 # equivalent: docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
 docker run -it -v $PWD:/data archivebox/archivebox help
-
-# optional: import your browser cookies into a persona for logged-in archiving
-archivebox persona create --import=chrome personal
-# supported: chrome/chromium/brave/edge (Chromium-based only)
-# use --profile to target a specific profile (e.g. Default, Profile 1)
-# re-running import merges/dedupes cookies.txt (by domain/path/name) but replaces chrome_user_data
 ```

 #### ArchiveBox Subcommands

 - `archivebox` `help`/`version` to see the list of available subcommands / currently installed version info
 - `archivebox` `setup`/`init`/`config`/`status`/`shell`/`manage` to administer your collection
- `archivebox` `add`/`schedule` to pull in fresh URLs from [bookmarks/history/RSS/etc.](#input-formats)
+- `archivebox` `add`/`oneshot`/`schedule` to pull in fresh URLs from [bookmarks/history/RSS/etc.](#input-formats)
 - `archivebox` `list`/`update`/`remove` to manage existing Snapshots in your collection

 <br/>
@@ -512,7 +509,7 @@ archivebox persona create --import=chrome personal
 # make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
 <br/>
 # archivebox [subcommand] [--help]
-archivebox init --install      # safe to run init multiple times (also how you update versions)
+archivebox init --setup      # safe to run init multiple times (also how you update versions)
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
 archivebox add --depth=1 'https://news.ycombinator.com'
@@ -529,7 +526,7 @@ archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
 # docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox init --install
+docker compose run archivebox init --setup
 docker compose run archivebox version
 docker compose run archivebox help
 docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
@@ -547,7 +544,7 @@ docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
 # make sure you create and cd into in a new empty directory first  
 <br/>
 # docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init --install
+docker run -v $PWD:/data -it archivebox/archivebox init --setup
 docker run -v $PWD:/data -it archivebox/archivebox version
 docker run -v $PWD:/data -it archivebox/archivebox help
 docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.ycombinator.com'
@@ -587,8 +584,7 @@ docker run -v $PWD:/data -it archivebox/archivebox archivebox manage createsuper
 docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 </code></pre>

-<sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
-<sup>Set <code>LISTEN_HOST</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
+<sup>Open <a href="http://localhost:8000"><code>http://localhost:8000</code></a> to see your server's Web UI ➡️</sup>
 <br/><br/>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Usage: Web UI</a> wiki. ➡️</i>
 <br/><br/>
@@ -692,7 +688,7 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a

 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.

-It also includes a built-in scheduled import feature with `archivebox schedule`, handled by the same orchestrator that powers `archivebox server`, so you can pull in URLs from RSS feeds and websites regularly without a separate cron container.
+It also includes a built-in scheduled import feature with `archivebox schedule` and browser bookmarklet, so you can pull in URLs from RSS feeds, websites, or the filesystem regularly/on-demand.

 <br/>

@@ -753,7 +749,7 @@ env CHROME_BINARY=chromium archivebox ...       # run with a one-off config
 <sub>These methods also work the same way when run inside Docker, see the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Docker Configuration</a> wiki page for details.</sub>
 </details><br/>

-The configuration is documented here: **[Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)**, and loaded from: [`archivebox/config/`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config/).
+The configuration is documented here: **[Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)**, and loaded here: [`archivebox/config.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config.py).

 <a name="most-common-options-to-tweak"></a>
 <details>
@@ -764,12 +760,16 @@ The configuration is documented here: **[Configuration Wiki](https://github.com/
 <br/>
 TIMEOUT=240                # default: 60    add more seconds on slower networks
 CHECK_SSL_VALIDITY=False   # default: True  False = allow saving URLs w/ bad SSL
+SAVE_ARCHIVE_DOT_ORG=False # default: True  False = disable Archive.org saving
+MAX_MEDIA_SIZE=1500m       # default: 750m  raise/lower youtubedl output size
 <br/>
 PUBLIC_INDEX=True          # default: True  whether anon users can view index
 PUBLIC_SNAPSHOTS=True      # default: True  whether anon users can view pages
 PUBLIC_ADD_VIEW=False      # default: False whether anon users can add new URLs
 <br/>
-USER_AGENT="Mozilla/5.0 ..."  # change this to get around bot blocking
+CHROME_USER_AGENT="Mozilla/5.0 ..."  # change these to get around bot blocking
+WGET_USER_AGENT="Mozilla/5.0 ..."
+CURL_USER_AGENT="Mozilla/5.0 ..."
 </code></pre>
 </details>
 <br/>
@@ -790,19 +790,7 @@ ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.c
 <p><em>TIP: For better security while running ArchiveBox, and to avoid polluting your host system with a bunch of sub-dependencies that you need to keep up-to-date,<strong>it is strongly recommended to use the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">⭐️ official Docker image</a></strong> which provides everything in an easy container with simple one-liner upgrades.</em></p>
 </blockquote>

-<ul>
-<li>Language: Python <code>&gt;=3.13</code></li>
-<li>Backend: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://django-ninja.dev/">Django-Ninja</a> for REST API</li>
-<li>Frontend: <a href="https://docs.djangoproject.com/en/6.0/ref/contrib/admin/">Django Admin</a> + Vanilla HTML, CSS, JS</li>
-<li>Web Server: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://github.com/django/daphne/"><code>daphne</code></a> (ASGI)</li>
-<li>Database: <a href="https://docs.djangoproject.com/en/6.0/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite3</code></li>
-<li>Job Queue: Custom orchestrator using <code>supervisord</code> for worker management</li>
-<li>Build/test/lint: <a href="https://github.com/astral-sh/uv"><code>uv</code></a> / <code>pyright</code>+<code>ty</code>+<code>pytest</code> / <code>ruff</code></li>
-<li>Subdependencies: <a href="https://github.com/ArchiveBox/abxpkg"><code>abxpkg</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
-</ul>
-
-
-These optional subdependencies used for archiving sites include:
+These optional dependencies used for archiving sites include:

 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/62a02155-05d7-4f3e-8de5-75a50a145c4f" alt="archivebox --version CLI output screenshot showing dependencies installed" width="330px" align="right" style="max-width: 100%;">

@@ -827,7 +815,7 @@ If not using Docker, make sure to keep the dependencies up-to-date yourself and
 # apt/brew/pip/etc install ... (see Quickstart instructions above)
 <br/>
 which -a archivebox    # see where you have installed archivebox
-archivebox install     # auto install all the extractors and extras
+archivebox setup       # auto install all the extractors and extras
 archivebox --version   # see info and check validity of installed dependencies
 </code></pre>
  
@@ -897,7 +885,7 @@ Each snapshot subfolder <code>data/archive/TIMESTAMP/</code> includes a static <

 ## Static Archive Exporting

-You can export your index as static HTML using `archivebox list` (so you can view it without an ArchiveBox server).
+You can create one-off archives of individual URLs with `archivebox oneshot`, or export your index as static HTML using `archivebox list` (so you can view it without an ArchiveBox server).

 <br/>
 <details>
@@ -907,7 +895,10 @@ You can export your index as static HTML using `archivebox list` (so you can vie
 <p><em>NOTE: These exports are not paginated, exporting many URLs or the entire archive at once may be slow. Use the filtering CLI flags on the <code>archivebox list</code> command to export specific Snapshots or ranges.</em></p>
 </blockquote>

-<pre lang="bash"><code style="white-space: pre-line"># archivebox list --help
+<pre lang="bash"><code style="white-space: pre-line"># do a one-off single URL archive wihout needing a data dir initialized
+archivebox oneshot 'https://example.com'
+
+# archivebox list --help
 archivebox list --html --with-headers > index.html     # export to static html table
 archivebox list --json --with-headers > index.json     # export to json blob
 archivebox list --csv=timestamp,url,title > index.csv  # export to csv spreadsheet
@@ -952,11 +943,18 @@ If you're importing pages with private content or URLs containing secret tokens
 archivebox add 'https://docs.google.com/document/d/12345somePrivateDocument'
 archivebox add 'https://vimeo.com/somePrivateVideo'

+# without first disabling saving to Archive.org:
+archivebox config --set SAVE_ARCHIVE_DOT_ORG=False  # disable saving all URLs in Archive.org
+
 # restrict the main index, Snapshot content, and Add Page to authenticated users as-needed:
 archivebox config --set PUBLIC_INDEX=False
 archivebox config --set PUBLIC_SNAPSHOTS=False
-archivebox config --set PUBLIC_ADD_VIEW=False
+archivebox config --set PUBLIC_ADD_VIEW=False 
 archivebox manage createsuperuser
+
+# if extra paranoid or anti-Google:
+archivebox config --set SAVE_FAVICON=False          # disable favicon fetching (it calls a Google API passing the URL's domain part only)
+archivebox config --set CHROME_BINARY=chromium      # ensure it's using Chromium instead of Chrome
 </code></pre>

 <blockquote>
@@ -1079,7 +1077,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <br/>

 <ul>
-<li><strong>ArchiveBox can use anywhere from ~1gb per 1000 Snapshots, to ~50gb per 1000 Snapshots</strong>, mostly dependent on whether you're saving video/audio using <code>YTDLP_ENABLED=True</code> and whether you lower <code>YTDLP_MAX_SIZE=750m</code>.</li>
+<li><strong>ArchiveBox can use anywhere from ~1gb per 1000 Snapshots, to ~50gb per 1000 Snapshots</strong>, mostly dependent on whether you're saving audio & video using <code>SAVE_MEDIA=True</code> and whether you lower <code>MEDIA_MAX_SIZE=750mb</code>.</li>
 <li>Disk usage can be reduced by using a compressed/<a href="https://www.ixsystems.com/blog/ixsystems-and-klara-systems-celebrate-valentines-day-with-a-heartfelt-donation-of-fast-dedupe-to-openzfs-and-truenas/">deduplicated</a> filesystem like <a href="https://www.reddit.com/r/zfs/comments/t9cexx/a_simple_real_world_zfs_compression_speed_an/">ZFS</a>/BTRFS, or by turning off extractors methods you don't need. You can also deduplicate content with a tool like <a href="https://github.com/adrianlopezroche/fdupes"><code>fdupes</code></a> or <a href="https://github.com/pauldreik/rdfind"><code>rdfind</code></a>.  
 </li>
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
@@ -1185,7 +1183,7 @@ ArchiveBox's stance is that duplication of other people's content is only ethica

 In the U.S., <a href="https://guides.library.oregonstate.edu/copyright/libraries">libraries, researchers, and archivists</a> are allowed to duplicate copyrighted materials under <a href="https://libguides.ala.org/copyright/fairuse">"fair use"</a> for <a href="https://guides.cuny.edu/cunyfairuse/librarians#:~:text=One%20of%20these%20specified%20conditions,may%20be%20liable%20for%20copyright">private study, scholarship, or research</a>. Archive.org's non-profit preservation work is <a href="https://blog.archive.org/2024/03/01/fair-use-in-action-at-the-internet-archive/">covered under fair use</a> in the US, and they properly handle <a href="https://cardozoaelj.com/2015/03/20/use-of-copyright-law-to-take-down-revenge-porn/">unethical content</a>/<a href="https://help.archive.org/help/rights/">DMCA</a>/<a href="https://gdpr.eu/right-to-be-forgotten/#:~:text=An%20individual%20has%20the%20right,that%20individual%20withdraws%20their%20consent.">GDPR</a> removal requests to maintain good standing in the eyes of the law.

-As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use software like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal council if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and changing your instance's branding using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir"><code>CUSTOM_TEMPLATES_DIR</code></a>).
+As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use sofware like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal council if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and changing your instance's branding using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir"><code>CUSTOM_TEMPLATES_DIR</code></a>).

 </details>
 <br/>
@@ -1214,7 +1212,7 @@ ArchiveBox tries to be a robust, set-and-forget archiving solution suitable for

 Not all content is suitable to be archived on a centralized, publicly accessible platform. Archive.org doesn't offer the ability to save things behind login walls for good reason, as the content may not have been intended for a public audience. ArchiveBox exists to fill that gap by letting everyone save what they have access to on an individual basis, and to encourage decentralized archiving that's less succeptible to censorship or natural disasters.

-By having users store their content locally or within their organizations, we can also save much larger portions of the internet than a centralized service has the disk capacity to handle. The eventual goal is to work towards federated archiving where users can share portions of their collections with each other, and with central archives on a case-by-case basis.
+By having users store their content locally or within their organizations, we can also save much larger portions of the internet than a centralized service has the disk capcity handle. The eventual goal is to work towards federated archiving where users can share portions of their collections with each other, and with central archives on a case-by-case basis.

 <h3>Comparison With Other Self-Hosted Archiving Options</h3>

@@ -1223,7 +1221,7 @@ ArchiveBox differentiates itself from [similar self-hosted projects](https://git

 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*

-*If you want more bookmark categorization and note-taking features, check out [Memex](https://github.com/WorldBrain/Memex), [Hoarder](https://github.com/hoarder-app/hoarder), [LinkWarden](https://github.com/linkwarden/linkwarden),  [Archivy](https://archivy.github.io/), or [LinkAce](https://www.linkace.org/).*
+*If you want more bookmark categorization and note-taking features, check out [Archivy](https://archivy.github.io/), [Memex](https://github.com/WorldBrain/Memex), [Polar](https://getpolarized.io/), or [LinkAce](https://www.linkace.org/).*

 *If you need more advanced recursive spider/crawling ability beyond `--depth=1`, check out [Browsertrix](https://github.com/webrecorder/browsertrix-crawler), [Photon](https://github.com/s0md3v/Photon), or [Scrapy](https://scrapy.org/) and pipe the outputted URLs into ArchiveBox.*

@@ -1309,8 +1307,8 @@ We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wik
 ## Developers

 - [Developer Documentation](https://github.com/ArchiveBox/ArchiveBox#archivebox-development)
- [Python API](https://docs.archivebox.io/)
- [REST API](https://demo.archivebox.io/api) (alpha)
+- [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha)
+- [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha)

 ## More Info

@@ -1337,49 +1335,36 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc

 **Python API Documentation:** https://docs.archivebox.io/en/dev/archivebox.html#module-archivebox.main

-**Internal Architecture Diagrams:** https://github.com/ArchiveBox/ArchiveBox/wiki/ArchiveBox-Architecture-Diagrams
-
-
 ### Setup the dev environment

 <details><summary><i>Click to expand...</i></summary>

-#### 1. Setup the monorepo
-
-First make sure you have `uv` installed: https://docs.astral.sh/uv/getting-started/installation/
+#### 1. Clone the main code repo (making sure to pull the submodules as well)

 ```bash
-git clone https://github.com/ArchiveBox/monorepo
-cd monorepo
-./bin/setup.sh
-
-# activate the monorepo venv, then cd into archivebox
-source .venv/bin/activate
-cd archivebox
+git clone --recurse-submodules https://github.com/ArchiveBox/ArchiveBox
+cd ArchiveBox
+git checkout dev  # or the branch you want to test
+git submodule update --init --recursive
+git pull --recurse-submodules
 ```

-Repos included in monorepo setup:
-
- https://github.com/ArchiveBox/abxbus
- https://github.com/ArchiveBox/abxpkg
- https://github.com/ArchiveBox/abx-plugins
- https://github.com/ArchiveBox/abx-dl
- https://github.com/ArchiveBox/ArchiveBox
-
-
 #### 2. Option A: Install the Python, JS, and system dependencies directly on your machine

 ```bash
-# Install ArchiveBox runtime dependencies
-mkdir -p data && cd data
-archivebox init
-archivebox install         # detect and install all extractor dependencies
+# Install ArchiveBox + python dependencies
+python3 -m venv .venv && source .venv/bin/activate && pip install -e '.[dev]'
+# or: pipenv install --dev && pipenv shell

-# Run the development server w/ autoreloading (but no bg workers)
-archivebox server --debug --reload 0.0.0.0:8000
+# Install node dependencies
+npm install
+# or
+archivebox setup

-# Run the production server (with bg workers but no autoreloading)
-archivebox server 0.0.0.0:8000
+# Check to see if anything is missing
+archivebox --version
+# install any missing dependencies manually, or use the helper script:
+./bin/setup.sh
 ```

 #### 2. Option B: Build the docker container and use that for development instead
@@ -1388,15 +1373,14 @@ archivebox server 0.0.0.0:8000
 # Optional: develop via docker by mounting the code dir into the container
 # if you edit e.g. ./archivebox/core/models.py on the docker host, runserver
 # inside the container will reload and pick up your changes
-./bin/build_docker.sh dev
-
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev init --install
-
-# Run the development server w/ autoreloading (but no bg workers)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
-
-# Run the production server (with bg workers but no autoreloading)
-docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server
+docker build . -t archivebox
+docker run -it \
+    -v $PWD/data:/data \
+    archivebox init --setup
+docker run -it -p 8000:8000 \
+    -v $PWD/data:/data \
+    -v $PWD/archivebox:/app/archivebox \
+    archivebox server 0.0.0.0:8000 --debug --reload

 # (remove the --reload flag and add the --nothreading flag when profiling with the django debug toolbar)
 # When using --reload, make sure any files you create can be read by the user in the Docker container, eg with 'chmod a+rX'.
@@ -1414,15 +1398,9 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 <details><summary><i>Click to expand...</i></summary>

 ```bash
-# set up persistent DEBUG=True for all runs
 archivebox config --set DEBUG=True
-
-# OR you can run a dev server with DEBUG=True in a few ways:
-archivebox server --debug --reload 0.0.0.0:8000
 # or
-archivebox server --debug 0.0.0.0:8000
-# or
-env DEBUG=True daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
+archivebox server --debug ...
 ```

 https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
@@ -1457,12 +1435,12 @@ services:

 # or with plain Docker:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
-docker run -it -v $PWD:/data archivebox:dev init
+docker run -it -v $PWD:/data archivebox:dev init --setup

 # or with pip:
 pip install 'git+https://github.com/pirate/ArchiveBox@dev'
 npm install 'git+https://github.com/ArchiveBox/ArchiveBox.git#dev'
-archivebox install
+archivebox init --setup
 ```

 </details>
@@ -1475,7 +1453,7 @@ archivebox install
 ./bin/lint.sh
 ./bin/test.sh
 ```
-(uses `ruff`, `pyright`, `ty`, and `pytest -s`)
+(uses `flake8`, `mypy`, and `pytest -s`)

 </details>

@@ -1519,9 +1497,13 @@ archivebox manage runscript testdata
 archivebox manage reset_db

 # use django-tui to interactively explore commands
-uv pip install django-tui
+pip install django-tui
 # ensure django-tui is in INSTALLED_APPS: core/settings.py
 archivebox manage tui
+
+# show python and JS package dependency trees
+pdm list --tree
+npm ls --all
 ```

 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/dc3e9f8c-9544-46e0-a7f0-30f571b72022" width="600px" alt="ArchiveBox ORM models relatinoship graph"/>
@@ -1537,11 +1519,37 @@ archivebox manage tui

 #### Contributing a new extractor

+<details><summary><i>Click to expand...</i></summary>

-Extractors are maintained in a separate repo here: https://github.com/ArchiveBox/abx-plugins (included in monorepo setup).
+<br/><br/>

-Copy a similar plugin as a template to modify, then open a new PR to add it in that repo.
+ArchiveBox [`extractors`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/media.py) are external binaries or Python/Node scripts that ArchiveBox runs to archive content on a page.

+Extractors take the URL of a page to archive, write their output to the filesystem `data/archive/TIMESTAMP/EXTRACTOR/...`, and return an [`ArchiveResult`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/models.py#:~:text=return%20qs-,class%20ArchiveResult,-(models.Model)%3A) entry which is saved to the database (visible on the `Log` page in the UI).
+
+*Check out how we added **[`archivebox/extractors/singlefile.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/singlefile.py)** as an example of the process: [Issue #399](https://github.com/ArchiveBox/ArchiveBox/issues/399) + [PR #403](https://github.com/ArchiveBox/ArchiveBox/pull/403).*
+
+<br/>
+
+
+**The process to contribute a new extractor is like this:**
+
+1. [Open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new?assignees=&labels=changes%3A+behavior%2Cstatus%3A+idea+phase&template=feature_request.md&title=Feature+Request%3A+...) with your propsoed implementation (please link to the pages of any new external dependencies you plan on using)
+2. Ensure any dependencies needed are easily installable via a package managers like `apt`, `brew`, `pip3`, `npm`
+   (Ideally, prefer to use external programs available via `pip3` or `npm`, however we do support using any binary installable via package manager that exposes a CLI/Python API and writes output to stdout or the filesystem.)
+3. Create a new file in [`archivebox/extractors/EXTRACTOR.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors) (copy an existing extractor like [`singlefile.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/extractors/singlefile.py) as a template)
+4. Add config settings to enable/disable any new dependencies and the extractor as a whole, e.g. `USE_DEPENDENCYNAME`, `SAVE_EXTRACTORNAME`, `EXTRACTORNAME_SOMEOTHEROPTION` in [`archivebox/config.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config.py)
+5. Add a preview section to [`archivebox/templates/core/snapshot.html`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/snapshot.html) to view the output, and a column to [`archivebox/templates/core/index_row.html`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/templates/core/index_row.html) with an icon for your extractor
+6. Add an integration test for your extractor in [`tests/test_extractors.py`](https://github.com/ArchiveBox/ArchiveBox/blob/dev/tests/test_extractors.py)
+7. [Submit your PR for review!](https://github.com/ArchiveBox/ArchiveBox/blob/dev/.github/CONTRIBUTING.md) 🎉
+8. Once merged, please document it in these places and anywhere else you see info about other extractors:
+  - https://github.com/ArchiveBox/ArchiveBox#output-formats
+  - https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles
+  - https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies
+
+<br/><br/>
+
+</details>

 #### Build the docs, pip package, and docker image

@@ -1554,6 +1562,8 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 # or individually:
 ./bin/build_docs.sh
 ./bin/build_pip.sh
+./bin/build_deb.sh
+./bin/build_brew.sh
 ./bin/build_docker.sh
 ```

@@ -1570,6 +1580,8 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 # or individually:
 ./bin/release_docs.sh
 ./bin/release_pip.sh
+./bin/release_deb.sh
+./bin/release_brew.sh
 ./bin/release_docker.sh
 ```

@@ -1596,14 +1608,14 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 <a href="https://github.com/sponsors/pirate"><img src="https://img.shields.io/badge/Github_Sponsors-%23B7CDFE.svg"/></a> &nbsp;
 <a href="https://www.patreon.com/theSquashSH"><img src="https://img.shields.io/badge/Patreon-%23DD5D76.svg"/></a> &nbsp;
 <a href="https://paypal.me/NicholasSweeting"><img src="https://img.shields.io/badge/Paypal-%23FFD141.svg"/></a> &nbsp;
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a> &nbsp;
-<a href="https://archivebox-shop.fourthwall.com/"><img src="https://img.shields.io/badge/Merch-%23903851.svg"/></a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"><img src="https://img.shields.io/badge/BTC%5CETH-%231a1a1a.svg"/></a>
 <br/>
 <sup><i>ArchiveBox operates as a US 501(c)(3) nonprofit <a href="https://en.wikipedia.org/wiki/Fiscal_sponsorship">FSP</a> (sponsored by <a href="https://hackclub.com/hcb?ref=donation">HCB</a>), <a href="https://hcb.hackclub.com/donations/start/archivebox">direct donations</a> are tax-deductible.</i></sup>
 <br/><br/>
 <a href="https://twitter.com/ArchiveBoxApp"><img src="https://img.shields.io/badge/Tweet-%40ArchiveBoxApp-blue.svg?style=flat"/></a>&nbsp;
 <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?style=flat&label=Star+on+Github"/></a>&nbsp;
-<a href="https://zulip.archivebox.io/"><img src="https://img.shields.io/badge/Join_Our_Community-Zulip_Forum-%23B7EDFE.svg"/></a><br/><br/>
+<a href="https://zulip.archivebox.io/"><img src="https://img.shields.io/badge/Join_Our_Community-Zulip_Forum-%23B7EDFE.svg"/></a><br/>
+<sup>ArchiveBox was started by <a href="https://docs.sweeting.me/s/blog#About">Nick Sweeting</a> in 2017, and has <a href="https://star-history.com/#archivebox/archivebox&Date">grown steadily</a> with help from our <a href="https://github.com/ArchiveBox/ArchiveBox/graphs/contributors">amazing contributors</a>.</sup>
 <hr/>
 <i>✨ Have spare CPU/disk/bandwidth after all your 网站存档爬 and want to help the world?<br/>Check out our <a href="https://github.com/ArchiveBox/good-karma-kit">Good Karma Kit</a>...</i>
 </div>
@@ -3,4 +3,4 @@ ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E
 select = F,E9,W
 max-line-length = 130
 max-complexity = 10
-exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv,data,data*
+exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv
@@ -0,0 +1 @@
+../LICENSE
@@ -1,137 +1,4 @@
-#!/usr/bin/env python3
-
-# Welcome to the ArchiveBox source code! Thanks for checking it out!
-#
-# "We are swimming upstream against a great torrent of disorganization.
-# In this, our main obligation is to establish arbitrary enclaves of order and system.
-# It is the greatest possible victory to be, to continue to be, and to have been.
-# No defeat can deprive us of the success of having existed for some moment of time
-# in a universe that seems indifferent to us."
-# --Norber Weiner
-
-__package__ = "archivebox"
-
-import os
-import sys
-from pathlib import Path
-from typing import Protocol, cast
-
-from abx_plugins import get_plugins_dir
+__package__ = 'archivebox'


-class _ReconfigurableStream(Protocol):
-    def reconfigure(self, *, line_buffering: bool) -> object: ...
-
-
-# Force unbuffered output for real-time logs
-if hasattr(sys.stdout, "reconfigure"):
-    cast(_ReconfigurableStream, sys.stdout).reconfigure(line_buffering=True)
-    cast(_ReconfigurableStream, sys.stderr).reconfigure(line_buffering=True)
-os.environ["PYTHONUNBUFFERED"] = "1"
-
-ASCII_LOGO = """
- █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
-██╔══██╗██╔══██╗██╔════╝██║  ██║██║██║   ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
-███████║██████╔╝██║     ███████║██║██║   ██║█████╗   ██████╔╝██║   ██║ ╚███╔╝ 
-██╔══██║██╔══██╗██║     ██╔══██║██║╚██╗ ██╔╝██╔══╝   ██╔══██╗██║   ██║ ██╔██╗ 
-██║  ██║██║  ██║╚██████╗██║  ██║██║ ╚████╔╝ ███████╗ ██████╔╝╚██████╔╝██╔╝ ██╗
-╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
-"""
-
-PACKAGE_DIR = Path(__file__).resolve().parent
-
-# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models
-# # Migrations reference models like 'machine.Binary' which need to be importable
-# if str(PACKAGE_DIR) not in sys.path:
-#     sys.path.append(str(PACKAGE_DIR))
-
-os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
-os.environ["TZ"] = "UTC"
-
-# detect ArchiveBox user's UID/GID based on data dir ownership
-from .config.permissions import drop_privileges  # noqa
-
-drop_privileges()
-
-from .misc.checks import check_not_root, check_not_inside_source_dir, check_io_encoding  # noqa
-
-check_not_root()
-check_not_inside_source_dir()
-check_io_encoding()
-
-# Install monkey patches for third-party libraries
-from .misc.monkey_patches import *  # noqa
-
-# Plugin directories
-BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
-USER_PLUGINS_DIR = (
-    Path(
-        os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or os.environ.get("USER_PLUGINS_DIR") or os.environ.get("DATA_DIR", os.getcwd()),
-    )
-    / "custom_plugins"
-)
-
-# These are kept for backwards compatibility with existing code
-# that checks for plugins. The new hook system uses discover_hooks()
-ALL_PLUGINS = {
-    "builtin": BUILTIN_PLUGINS_DIR,
-    "user": USER_PLUGINS_DIR,
-}
-LOADED_PLUGINS = ALL_PLUGINS
-
-# Setup basic config, constants, paths, and version
-from .config.constants import CONSTANTS  # noqa
-from .config.paths import PACKAGE_DIR, DATA_DIR  # noqa
-from .config.version import VERSION  # noqa
-
-# Set MACHINE_ID env var so hook scripts can use it
-os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
-
-__version__ = VERSION
-__author__ = "ArchiveBox"
-__license__ = "MIT"
-
-ASCII_ICON = """
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-         ██                                                                            ██          
-         ██                                                                            ██        
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                    ████████████████████████████████████                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ████████████████████████████████████                    ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                 ██████████████████████████████████████████                 ██          
-         ██                 ██████████████████████████████████████████                 ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██        
-         ████████████████████████████████████████████████████████████████████████████████          
-"""
+from .monkey_patches import *
@@ -1,20 +1,11 @@
 #!/usr/bin/env python3
-"""This is the entrypoint for python -m archivebox ..."""

-__package__ = "archivebox"
+__package__ = 'archivebox'

-import archivebox  # noqa # make sure monkey patches are applied before anything else
 import sys

 from .cli import main

-ASCII_LOGO_MINI = r"""
-     _             _     _           ____
-    / \   _ __ ___| |__ (_)_   _____| __ )  _____  __
-   / _ \ | '__/ __| '_ \| \ \ / / _ \  _ \ / _ \ \/ /
-  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  <
- /_/   \_\_|  \___|_| |_|_| \_/ \___|____/ \___/_/\_\
-"""

-if __name__ == "__main__":
+if __name__ == '__main__':
    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -0,0 +1 @@
+__package__ = 'abid_utils'
@@ -0,0 +1,202 @@
+from typing import NamedTuple, Any, Union, Optional
+
+import ulid
+import uuid6
+import hashlib
+from urllib.parse import urlparse
+
+from uuid import UUID
+from typeid import TypeID            # type: ignore[import-untyped]
+from datetime import datetime
+
+
+
+ABID_PREFIX_LEN = 4
+ABID_SUFFIX_LEN = 26
+ABID_LEN = 30
+ABID_TS_LEN = 10
+ABID_URI_LEN = 8
+ABID_SUBTYPE_LEN = 2
+ABID_RAND_LEN = 6
+
+DEFAULT_ABID_PREFIX = 'obj_'
+
+# allows people to keep their uris secret on a per-instance basis by changing the salt.
+# the default means everyone can share the same namespace for URI hashes,
+# meaning anyone who has a URI and wants to check if you have it can guess the ABID
+DEFAULT_ABID_URI_SALT = '687c2fff14e3a7780faa5a40c237b19b5b51b089'
+
+
+class ABID(NamedTuple):
+    """
+    e.g. ABID('obj_01HX9FPYTRE4A5CCD901ZYEBQE')
+    """
+    prefix: str            # e.g. obj_
+    ts: str                # e.g. 01HX9FPYTR
+    uri: str               # e.g. E4A5CCD9
+    subtype: str           # e.g. 01
+    rand: str              # e.g. ZYEBQE
+    
+    # salt: str = DEFAULT_ABID_URI_SALT
+
+    def __getattr__(self, attr: str) -> Any:
+        return getattr(self.ulid, attr)
+
+    def __eq__(self, other: Any) -> bool:
+        try:
+            return self.ulid == other.ulid
+        except AttributeError:
+            return NotImplemented
+
+    def __str__(self) -> str:
+        return self.prefix + self.suffix
+
+    def __len__(self) -> int:
+        return len(self.prefix + self.suffix)
+
+    @classmethod
+    def parse(cls, buffer: Union[str, UUID, ulid.ULID, TypeID, 'ABID'], prefix=DEFAULT_ABID_PREFIX) -> 'ABID':
+        assert buffer, f'Attempted to create ABID from null value {buffer}'
+
+        buffer = str(buffer)
+        if '_' in buffer:
+            prefix, suffix = buffer.split('_')
+        else:
+            prefix, suffix = prefix.strip('_'), buffer
+
+        assert len(prefix) == ABID_PREFIX_LEN - 1   # length without trailing _
+        assert len(suffix) == ABID_SUFFIX_LEN, f'Suffix {suffix} from {buffer} was not {ABID_SUFFIX_LEN} chars long'
+
+        return cls(
+            prefix=abid_part_from_prefix(prefix),
+            ts=suffix[0:10].upper(),
+            uri=suffix[10:18].upper(),
+            subtype=suffix[18:20].upper(),
+            rand=suffix[20:26].upper(),
+        )
+    
+    @property
+    def uri_salt(self) -> str:
+        return DEFAULT_ABID_URI_SALT
+
+    @property
+    def suffix(self):
+        return ''.join((self.ts, self.uri, self.subtype, self.rand))
+    
+    @property
+    def ulid(self) -> ulid.ULID:
+        return ulid.parse(self.suffix)
+
+    @property
+    def uuid(self) -> UUID:
+        return self.ulid.uuid
+
+    @property
+    def uuid6(self) -> uuid6.UUID:
+        return uuid6.UUID(hex=self.uuid.hex)
+
+    @property
+    def typeid(self) -> TypeID:
+        return TypeID.from_uuid(prefix=self.prefix.strip('_'), suffix=self.uuid6)
+
+    @property
+    def datetime(self) -> datetime:
+        return self.ulid.timestamp().datetime
+
+
+
+####################################################
+
+
+def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
+    """
+    'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25'
+    """
+    if isinstance(uri, bytes):
+        uri_str: str = uri.decode()
+    else:
+        uri_str = str(uri)
+
+    # only hash the domain part of URLs
+    if '://' in uri_str:
+        try:
+            domain = urlparse(uri_str).netloc
+            if domain:
+                uri_str = domain
+        except AttributeError:
+            pass
+    
+    uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
+
+    return hashlib.sha256(uri_bytes).hexdigest().upper()
+
+def abid_part_from_prefix(prefix: Optional[str]) -> str:
+    """
+    'snp_'
+    """
+    if prefix is None:
+        return 'obj_'
+
+    prefix = prefix.strip('_').lower()
+    assert len(prefix) == 3
+    return prefix + '_'
+
+def abid_part_from_uri(uri: str, salt: str=DEFAULT_ABID_URI_SALT) -> str:
+    """
+    'E4A5CCD9'     # takes first 8 characters of sha256(url)
+    """
+    uri = str(uri)
+    return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
+
+def abid_part_from_ts(ts: Optional[datetime]) -> str:
+    """
+    '01HX9FPYTR'   # produces 10 character Timestamp section of ulid based on added date
+    """
+    return str(ulid.from_timestamp(ts) if ts else ulid.new())[:ABID_TS_LEN]
+
+def abid_part_from_subtype(subtype: str) -> str:
+    """
+    Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
+    Also allows us to change the ulid spec later by putting special sigil values here.
+    """
+    subtype = str(subtype)
+    if len(subtype) == ABID_SUBTYPE_LEN:
+        return subtype
+
+    return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()
+
+def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
+    """
+    'ZYEBQE'   # takes last 6 characters of randomness from existing legacy uuid db field
+    """
+    if rand is None:
+        # if it's None we generate a new random 6 character hex string
+        return str(ulid.new())[-ABID_RAND_LEN:]
+    elif isinstance(rand, UUID):
+        # if it's a uuid we take the last 6 characters of the ULID represation of it
+        return str(ulid.from_uuid(rand))[-ABID_RAND_LEN:]
+    elif isinstance(rand, int):
+        # if it's a BigAutoInteger field we convert it from an int to a 0-padded string
+        rand_str = str(rand)[-ABID_RAND_LEN:]
+        padding_needed = ABID_RAND_LEN - len(rand_str)
+        rand_str = ('0'*padding_needed) + rand_str
+        return rand_str
+
+    # otherwise treat it as a string, take the last 6 characters of it verbatim
+    return str(rand)[-ABID_RAND_LEN:].upper()
+
+
+def abid_from_values(prefix, ts, uri, subtype, rand, salt=DEFAULT_ABID_URI_SALT) -> ABID:
+    """
+    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+    """
+
+    abid = ABID(
+        prefix=abid_part_from_prefix(prefix),
+        ts=abid_part_from_ts(ts),
+        uri=abid_part_from_uri(uri, salt=salt),
+        subtype=abid_part_from_subtype(subtype),
+        rand=abid_part_from_rand(rand),
+    )
+    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
+    return abid
@@ -0,0 +1,18 @@
+from django.contrib import admin
+
+
+class ABIDModelAdmin(admin.ModelAdmin):
+    list_display = ('created', 'created_by', 'abid', '__str__')
+    sort_fields = ('created', 'created_by', 'abid', '__str__')
+    readonly_fields = ('abid', 'created', '__str__')
+
+    def get_form(self, request, obj=None, **kwargs):
+        form = super().get_form(request, obj, **kwargs)
+        if 'created_by' in form.base_fields:
+            form.base_fields['created_by'].initial = request.user
+        return form
+
+    # def save_model(self, request, obj, form, change):
+    #     if getattr(obj, 'created_by_id', None) in (None, get_or_create_system_user_pk()):
+    #         obj.created_by = request.user
+    #     obj.save()
@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class AbidUtilsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    
+    name = 'abid_utils'
@@ -0,0 +1,344 @@
+"""
+This file provides the Django ABIDField and ABIDModel base model to inherit from.
+
+It implements the ArchiveBox ID (ABID) interfaces including abid_values, generate_abid, .abid, .uuid, .id.
+"""
+
+from typing import Any, Dict, Union, List, Set, NamedTuple, cast
+
+from ulid import ULID
+from uuid import uuid4, UUID
+from typeid import TypeID            # type: ignore[import-untyped]
+from datetime import datetime
+from functools import partial
+from charidfield import CharIDField  # type: ignore[import-untyped]
+
+from django.conf import settings
+from django.db import models
+from django.utils import timezone
+from django.db.utils import OperationalError
+from django.contrib.auth import get_user_model
+
+from django_stubs_ext.db.models import TypedModelMeta
+
+from .abid import (
+    ABID,
+    ABID_LEN,
+    ABID_RAND_LEN,
+    ABID_SUFFIX_LEN,
+    DEFAULT_ABID_PREFIX,
+    DEFAULT_ABID_URI_SALT,
+    abid_part_from_prefix,
+    abid_from_values
+)
+
+####################################################
+
+
+# Database Field for typeid/ulid style IDs with a prefix, e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ
+ABIDField = partial(
+    CharIDField,
+    max_length=ABID_LEN,
+    help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
+    default=None,
+    null=True,
+    blank=True,
+    db_index=True,
+    unique=True,
+)
+
+def get_or_create_system_user_pk(username='system'):
+    """Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
+
+    User = get_user_model()
+
+    # if only one user exists total, return that user
+    if User.objects.filter(is_superuser=True).count() == 1:
+        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
+
+    # otherwise, create a dedicated "system" user
+    user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
+    return user.pk
+
+
+class ABIDModel(models.Model):
+    """
+    Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
+    """
+    abid_prefix: str = DEFAULT_ABID_PREFIX  # e.g. 'tag_'
+    abid_ts_src = 'None'                    # e.g. 'self.created'
+    abid_uri_src = 'None'                   # e.g. 'self.uri'
+    abid_subtype_src = 'None'               # e.g. 'self.extractor'
+    abid_rand_src = 'None'                  # e.g. 'self.uuid' or 'self.id'
+
+    # id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
+    # uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    def save(self, *args: Any, **kwargs: Any) -> None:
+        # when first creating a row, self.ABID is the source of truth
+        # overwrite default prefilled self.id & self.abid with generated self.ABID value
+        if self._state.adding or not self.id:
+            self.id = self.ABID.uuid
+        if self._state.adding or not self.abid:
+            self.abid = str(self.ABID)
+
+        super().save(*args, **kwargs)
+        assert str(self.id) == str(self.ABID.uuid), f'self.id {self.id} does not match self.ABID {self.ABID.uuid}'
+        assert str(self.abid) == str(self.ABID), f'self.abid {self.id} does not match self.ABID {self.ABID.uuid}'
+
+    @property
+    def abid_values(self) -> Dict[str, Any]:
+        return {
+            'prefix': self.abid_prefix,
+            'ts': eval(self.abid_ts_src),
+            'uri': eval(self.abid_uri_src),
+            'subtype': eval(self.abid_subtype_src),
+            'rand': eval(self.abid_rand_src),
+        }
+
+    def generate_abid(self) -> ABID:
+        """
+        Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+        """
+        prefix, ts, uri, subtype, rand = self.abid_values.values()
+
+        if (not prefix) or prefix == DEFAULT_ABID_PREFIX:
+            suggested_abid = self.__class__.__name__[:3].lower()
+            raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
+
+        if not ts:
+            # default to unix epoch with 00:00:00 UTC
+            ts = datetime.fromtimestamp(0, timezone.utc)     # equivalent to: ts = datetime.utcfromtimestamp(0)
+            print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
+
+        if not uri:
+            uri = str(self)
+            print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
+
+        if not subtype:
+            subtype = self.__class__.__name__
+            print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
+
+        if not rand:
+            rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
+            print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
+
+        abid = abid_from_values(
+            prefix=prefix,
+            ts=ts,
+            uri=uri,
+            subtype=subtype,
+            rand=rand,
+            salt=DEFAULT_ABID_URI_SALT,
+        )
+        assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
+        return abid
+
+    @property
+    def ABID(self) -> ABID:
+        """
+        ULIDParts(timestamp='01HX9FPYTR', url='E4A5CCD9', subtype='00', randomness='ZYEBQE')
+        """
+        
+        # if object is not yet saved to DB, always generate fresh ABID from values
+        if self._state.adding:
+            return self.generate_abid()
+        
+        # otherwise DB is single source of truth, load ABID from existing db pk
+        abid: ABID | None = None
+        try:
+            abid = abid or ABID.parse(self.pk)
+        except Exception:
+            pass
+
+        try:
+            abid = abid or ABID.parse(self.id)
+        except Exception:
+            pass
+
+        try:
+            abid = abid or ABID.parse(cast(str, self.abid))
+        except Exception:
+            pass
+
+        abid = abid or self.generate_abid()
+
+        return abid
+
+    @property
+    def ULID(self) -> ULID:
+        """
+        Get a ulid.ULID representation of the object's ABID.
+        """
+        return self.ABID.ulid
+
+    @property
+    def UUID(self) -> UUID:
+        """
+        Get a uuid.UUID (v4) representation of the object's ABID.
+        """
+        return self.ABID.uuid
+
+    @property
+    def TypeID(self) -> TypeID:
+        """
+        Get a typeid.TypeID (stripe-style) representation of the object's ABID.
+        """
+        return self.ABID.typeid
+
+
+
+####################################################
+
+# Django helpers
+def find_all_abid_prefixes() -> Dict[str, type[models.Model]]:
+    """
+    Return the mapping of all ABID prefixes to their models.
+    e.g. {'tag_': core.models.Tag, 'snp_': core.models.Snapshot, ...}
+    """
+    import django.apps
+    prefix_map = {}
+
+    for model in django.apps.apps.get_models():
+        abid_prefix = getattr(model, 'abid_prefix', None)
+        if abid_prefix:
+            prefix_map[abid_prefix] = model
+    return prefix_map
+
+def find_prefix_for_abid(abid: ABID) -> str:
+    """
+    Find the correct prefix for a given ABID that may have be missing a prefix (slow).
+    e.g. ABID('obj_01BJQMF54D093DXEAWZ6JYRPAQ') -> 'snp_'
+    """
+    # if existing abid prefix is correct, lookup is easy
+    model = find_model_from_abid(abid)
+    if model:
+        assert issubclass(model, ABIDModel)
+        return model.abid_prefix
+
+    # prefix might be obj_ or missing, fuzzy-search to find any object that matches
+    return find_obj_from_abid_rand(abid)[0].abid_prefix
+
+def find_model_from_abid_prefix(prefix: str) -> type[ABIDModel] | None:
+    """
+    Return the Django Model that corresponds to a given ABID prefix.
+    e.g. 'tag_' -> core.models.Tag
+    """
+    prefix = abid_part_from_prefix(prefix)
+
+    import django.apps
+
+    for model in django.apps.apps.get_models():
+        if not issubclass(model, ABIDModel): continue   # skip non-ABID-enabled models
+        if not hasattr(model, 'objects'): continue      # skip abstract models
+
+        if (model.abid_prefix == prefix):
+            return model
+
+    return None
+
+def find_model_from_abid(abid: ABID) -> type[models.Model] | None:
+    """
+    Shortcut for find_model_from_abid_prefix(abid.prefix)
+    """
+    return find_model_from_abid_prefix(abid.prefix)
+
+def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDModel]:
+    """
+    Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
+    e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
+    """
+
+    # convert str to ABID if necessary
+    if isinstance(rand, ABID):
+        abid: ABID = rand
+    else:
+        rand = str(rand)
+        if len(rand) < ABID_SUFFIX_LEN:
+            padding_needed = ABID_SUFFIX_LEN - len(rand)
+            rand = ('0'*padding_needed) + rand
+        abid = ABID.parse(rand)
+
+    import django.apps
+
+    partial_matches: List[ABIDModel] = []
+
+    models_to_try = cast(Set[type[models.Model]], set(filter(bool, (
+        model,
+        find_model_from_abid(abid),
+        *django.apps.apps.get_models(),
+    ))))
+    # print(abid, abid.rand, abid.uuid, models_to_try)
+
+    for model in models_to_try:
+        if not issubclass(model, ABIDModel): continue   # skip Models that arent ABID-enabled
+        if not hasattr(model, 'objects'): continue      # skip abstract Models
+        assert hasattr(model, 'objects')                # force-fix for type hint nit about missing manager https://github.com/typeddjango/django-stubs/issues/1684
+
+        # continue on to try fuzzy searching by randomness portion derived from uuid field
+        try:
+            qs = []
+            if hasattr(model, 'abid'):
+                qs = model.objects.filter(abid__endswith=abid.rand)
+            elif hasattr(model, 'uuid'):
+                qs = model.objects.filter(uuid__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
+            elif hasattr(model, 'id'):
+                # NOTE: this only works on SQLite where every column is a string
+                # other DB backends like postgres dont let you do __endswith if this is a BigAutoInteger field
+                
+                # try to search for uuid=...-2354352
+                # try to search for id=...2354352
+                # try to search for id=2354352
+                qs = model.objects.filter(
+                    models.Q(id__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
+                    | models.Q(id__endswith=abid.rand)
+                    | models.Q(id__startswith=str(int(abid.rand)) if abid.rand.isdigit() else abid.rand)
+                )
+
+            for obj in qs:
+                if obj.generate_abid() == abid:
+                    # found exact match, no need to keep iterating
+                    return [obj]
+                partial_matches.append(obj)
+        except OperationalError as err:
+            print(f'[!] WARNING: Got error while trying to iterate through QuerySet for {model}:', err, '\n')
+
+    return partial_matches
+
+def find_obj_from_abid(abid: ABID, model=None, fuzzy=False) -> Any:
+    """
+    Find an object with a given ABID by filtering possible models for a matching abid/uuid/id (fast).
+    e.g. 'snp_01BJQMF54D093DXEAWZ6JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
+    """
+
+    model = model or find_model_from_abid(abid)
+    assert model, f'Could not find model that could match this ABID type: {abid}'
+
+    try:
+        if hasattr(model, 'abid'):
+            return model.objects.get(abid__endswith=abid.suffix)
+        if hasattr(model, 'uuid'):
+            return model.objects.get(uuid=abid.uuid)
+        return model.objects.get(id=abid.uuid)
+    except model.DoesNotExist:
+        # if the model has an abid field then it shouldve matched, pointless to fuzzy search in that case
+        if hasattr(model, 'abid') or (not fuzzy):
+            raise
+
+    # continue on to try fuzzy searching by randomness portion derived from uuid field
+    match_by_rand = find_obj_from_abid_rand(abid, model=model)
+    if match_by_rand:
+        if match_by_rand[0].abid_prefix != abid.prefix:
+            print(f'[!] WARNING: fetched object {match_by_rand} even though prefix {abid.prefix} doesnt match!', abid, '\n')
+        return match_by_rand
+
+    raise model.DoesNotExist
+
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
@@ -1 +1 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'
@@ -1,120 +0,0 @@
-__package__ = "archivebox.api"
-
-from django import forms
-from django.contrib import admin
-from django.http import HttpRequest
-from django.utils.text import capfirst
-from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
-from signal_webhooks.settings import webhook_settings
-from signal_webhooks.utils import get_webhook_model, model_from_reference
-
-from archivebox.base_models.admin import BaseModelAdmin
-
-from archivebox.api.models import APIToken
-
-
-def _webhook_fields(*names: str) -> tuple[str, ...]:
-    model_fields = {field.name for field in get_webhook_model()._meta.fields}
-    return tuple(name for name in names if name in model_fields)
-
-
-class APITokenAdmin(BaseModelAdmin):
-    list_display = ("created_at", "id", "created_by", "token_redacted", "expires")
-    sort_fields = ("id", "created_at", "created_by", "expires")
-    readonly_fields = ("created_at", "modified_at")
-    search_fields = ("id", "created_by__username", "token")
-
-    fieldsets = (
-        (
-            "Token",
-            {
-                "fields": ("token", "expires"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Owner",
-            {
-                "fields": ("created_by",),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": ("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
-    )
-
-    list_filter = ("created_by",)
-    ordering = ["-created_at"]
-    list_per_page = 100
-
-
-class OutboundWebhookAdminForm(WebhookModelForm):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.fields["ref"] = forms.ChoiceField(
-            label=self.fields["ref"].label,
-            help_text=self.fields["ref"].help_text,
-            choices=[
-                (ref, f"{capfirst(model_from_reference(ref, check_hooks=False)._meta.verbose_name_plural)} ({ref})")
-                for ref in sorted(webhook_settings.HOOKS)
-            ],
-        )
-
-
-class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
-    form = OutboundWebhookAdminForm
-    list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
-    sort_fields = _webhook_fields("created_at", "created_by", "id", "ref", "endpoint", "last_success", "last_failure")
-    readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
-
-    fieldsets = (
-        (
-            "Webhook",
-            {
-                "fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"),
-                "classes": ("card", "wide"),
-            },
-        ),
-        (
-            "Authentication",
-            {
-                "fields": _webhook_fields("auth_token"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Status",
-            {
-                "fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Owner",
-            {
-                "fields": _webhook_fields("created_by"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": _webhook_fields("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
-    )
-
-    def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:
-        """Preserve WebhookAdmin's auth token filter with Django's current admin signature."""
-        return not lookup.startswith("auth_token") and admin.ModelAdmin.lookup_allowed(self, lookup, value, request)
-
-
-def register_admin(admin_site: admin.AdminSite) -> None:
-    admin_site.register(APIToken, APITokenAdmin)
-    admin_site.register(get_webhook_model(), CustomWebhookAdmin)
@@ -1,14 +1,11 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

 from django.apps import AppConfig


+
 class APIConfig(AppConfig):
-    name = "archivebox.api"
-    label = "api"
+    name = 'api'

-
-def register_admin(admin_site):
-    from archivebox.api.admin import register_admin
-
-    register_admin(admin_site)
+    def ready(self):
+        pass
@@ -1,136 +1,102 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from datetime import timedelta
+from typing import Optional, cast

-from django.utils import timezone
 from django.http import HttpRequest
+from django.contrib.auth import login
 from django.contrib.auth import authenticate
-from django.contrib.auth.models import User
+from django.contrib.auth.models import AbstractBaseUser

-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth
-from ninja.errors import HttpError
+from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser


-def get_or_create_api_token(user: User | None):
-    from archivebox.api.models import APIToken
-
-    if user and user.is_superuser:
-        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
-        if api_tokens.exists():
-            # unexpired token exists, use it
-            api_token = api_tokens.last()
-        else:
-            # does not exist, create a new one
-            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
-
-        if api_token is None:
-            return None
-        assert api_token.is_valid(), f"API token is not valid {api_token}"
-
-        return api_token
-    return None
-
-
-def auth_using_token(token: str | None, request: HttpRequest | None = None) -> User | None:
+def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
    """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
-    from archivebox.api.models import APIToken  # lazy import model to avoid loading it at urls.py import time
+    from api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
+    
+    user = None

-    user: User | None = None
-
-    submitted_empty_form = str(token).strip() in ("string", "", "None", "null")
-    if not submitted_empty_form:
+    submitted_empty_form = token in ('string', '', None)
+    if submitted_empty_form:
+        assert request is not None, 'No request provided for API key authentication'
+        user = request.user       # see if user is authed via django session and use that as the default
+    else:
        try:
-            api_token = APIToken.objects.get(token=token)
-            if api_token.is_valid() and isinstance(api_token.created_by, User):
-                user = api_token.created_by
-                if request is not None:
-                    setattr(request, "_api_token", api_token)
+            token = APIToken.objects.get(token=token)
+            if token.is_valid():
+                user = token.created_by
        except APIToken.DoesNotExist:
            pass

-    return user
+    if not user:
+        print('[❌] Failed to authenticate API user using API Key:', request)

+    return None

-def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
+def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
    """Given a username and password, check if they are valid and return the corresponding user"""
-    user: User | None = None
-
-    submitted_empty_form = (username, password) in (("string", "string"), ("", ""), (None, None))
-    if not submitted_empty_form:
-        authenticated_user = authenticate(
+    user = None
+    
+    submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
+    if submitted_empty_form:
+        assert request is not None, 'No request provided for API key authentication'
+        user = request.user       # see if user is authed via django session and use that as the default
+    else:
+        user = authenticate(
            username=username,
            password=password,
        )
-        if isinstance(authenticated_user, User):
-            user = authenticated_user
-    return user
+
+    if not user:
+        print('[❌] Failed to authenticate API user using API Key:', request)
+        user = None
+
+    return cast(AbstractBaseUser | None, user)


 ### Base Auth Types

+class APITokenAuthCheck:
+    """The base class for authentication methods that use an api.models.APIToken"""
+    def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
+        user = auth_using_token(
+            token=key,
+            request=request,
+        )
+        if user is not None:
+            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
+        return user

-def _require_superuser(user: User | None, request: HttpRequest, auth_method: str) -> User | None:
-    if user and user.pk:
-        request.user = user
-        setattr(request, "_api_auth_method", auth_method)
-        if not user.is_superuser:
-            raise HttpError(403, "Valid credentials but User does not have permission (make sure user.is_superuser=True)")
-    return user
+class UserPassAuthCheck:
+    """The base class for authentication methods that use a username & password"""
+    def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
+        user = auth_using_password(
+            username=username,
+            password=password,
+            request=request,
+        )
+        if user is not None:
+            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
+        return user


 ### Django-Ninja-Provided Auth Methods

-
-class HeaderTokenAuth(APIKeyHeader):
+class HeaderTokenAuth(APITokenAuthCheck, APIKeyHeader):
    """Allow authenticating by passing X-API-Key=xyz as a request header"""
-
    param_name = "X-ArchiveBox-API-Key"

-    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
-        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
-
-
-class BearerTokenAuth(HttpBearer):
+class BearerTokenAuth(APITokenAuthCheck, HttpBearer):
    """Allow authenticating by passing Bearer=xyz as a request header"""
+    pass

-    def authenticate(self, request: HttpRequest, token: str) -> User | None:
-        return _require_superuser(auth_using_token(token=token, request=request), request, self.__class__.__name__)
-
-
-class QueryParamTokenAuth(APIKeyQuery):
+class QueryParamTokenAuth(APITokenAuthCheck, APIKeyQuery):
    """Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
-
    param_name = "api_key"

-    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
-        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
-
-
-class UsernameAndPasswordAuth(HttpBasicAuth):
+class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
-
-    def authenticate(self, request: HttpRequest, username: str, password: str) -> User | None:
-        return _require_superuser(
-            auth_using_password(username=username, password=password, request=request),
-            request,
-            self.__class__.__name__,
-        )
-
-
-class DjangoSessionAuth:
-    """Allow authenticating with existing Django session cookies (same-origin only)."""
-
-    def __call__(self, request: HttpRequest) -> User | None:
-        return self.authenticate(request)
-
-    def authenticate(self, request: HttpRequest, **kwargs) -> User | None:
-        user = getattr(request, "user", None)
-        if isinstance(user, User) and user.is_authenticated:
-            setattr(request, "_api_auth_method", self.__class__.__name__)
-            if not user.is_superuser:
-                raise HttpError(403, "Valid session but User does not have permission (make sure user.is_superuser=True)")
-            return user
-        return None
+    pass


 ### Enabled Auth Methods
@@ -138,6 +104,7 @@ class DjangoSessionAuth:
 API_AUTH_METHODS = [
    HeaderTokenAuth(),
    BearerTokenAuth(),
-    QueryParamTokenAuth(),
-    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
+    QueryParamTokenAuth(), 
+    django_auth_superuser,
+    UsernameAndPasswordAuth(),
 ]
@@ -1,32 +0,0 @@
-__package__ = "archivebox.api"
-
-from django.http import HttpResponse
-
-
-class ApiCorsMiddleware:
-    """Attach permissive CORS headers for API routes (token-based auth)."""
-
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        if request.path.startswith("/api/"):
-            if request.method == "OPTIONS" and request.META.get("HTTP_ACCESS_CONTROL_REQUEST_METHOD"):
-                response = HttpResponse(status=204)
-                return self._add_cors_headers(request, response)
-
-            response = self.get_response(request)
-            return self._add_cors_headers(request, response)
-
-        return self.get_response(request)
-
-    def _add_cors_headers(self, request, response):
-        origin = request.META.get("HTTP_ORIGIN")
-        if not origin:
-            return response
-
-        response["Access-Control-Allow-Origin"] = "*"
-        response["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
-        response["Access-Control-Allow-Headers"] = "Authorization, X-ArchiveBox-API-Key, Content-Type, X-CSRFToken"
-        response["Access-Control-Max-Age"] = "600"
-        return response
@@ -1,239 +1,29 @@
-# Generated by hand on 2025-12-29
-# Creates APIToken and OutboundWebhook tables using raw SQL
+# Generated by Django 4.2.11 on 2024-04-25 04:19

+import api.models
+from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
-import django.utils.timezone
-from django.conf import settings
-from archivebox.uuid_compat import uuid7
-from archivebox.base_models.models import get_or_create_system_user_pk
-import archivebox.api.models
-import signal_webhooks.fields
-import signal_webhooks.utils
+import uuid


 class Migration(migrations.Migration):
+
    initial = True

    dependencies = [
-        ("auth", "0012_alter_user_first_name_max_length"),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
-        migrations.SeparateDatabaseAndState(
-            database_operations=[
-                migrations.RunSQL(
-                    sql="""
-                -- Create api_apitoken table
-                CREATE TABLE IF NOT EXISTS api_apitoken (
-                    id TEXT PRIMARY KEY NOT NULL,
-                    created_at DATETIME NOT NULL,
-                    modified_at DATETIME NOT NULL,
-
-                    token VARCHAR(32) NOT NULL UNIQUE,
-                    expires DATETIME,
-
-                    created_by_id INTEGER NOT NULL,
-
-                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
-                );
-                CREATE INDEX IF NOT EXISTS api_apitoken_created_by_id_idx ON api_apitoken(created_by_id);
-                CREATE INDEX IF NOT EXISTS api_apitoken_created_at_idx ON api_apitoken(created_at);
-                CREATE INDEX IF NOT EXISTS api_apitoken_token_idx ON api_apitoken(token);
-
-                -- Create api_outboundwebhook table
-                CREATE TABLE IF NOT EXISTS api_outboundwebhook (
-                    id TEXT PRIMARY KEY NOT NULL,
-                    created_at DATETIME NOT NULL,
-                    modified_at DATETIME NOT NULL,
-
-                    name VARCHAR(255) NOT NULL UNIQUE,
-                    signal VARCHAR(255) NOT NULL,
-                    ref VARCHAR(1024) NOT NULL,
-                    endpoint VARCHAR(2048) NOT NULL,
-                    headers TEXT NOT NULL DEFAULT '{}',
-                    auth_token TEXT NOT NULL DEFAULT '',
-                    enabled BOOLEAN NOT NULL DEFAULT 1,
-                    keep_last_response BOOLEAN NOT NULL DEFAULT 0,
-                    created DATETIME NOT NULL,
-                    updated DATETIME NOT NULL,
-                    last_response TEXT NOT NULL DEFAULT '',
-                    last_success DATETIME,
-                    last_failure DATETIME,
-
-                    created_by_id INTEGER NOT NULL,
-
-                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
-                );
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_by_id_idx ON api_outboundwebhook(created_by_id);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
-                    """,
-                    reverse_sql="""
-                DROP TABLE IF EXISTS api_outboundwebhook;
-                DROP TABLE IF EXISTS api_apitoken;
-                    """,
-                ),
-            ],
-            state_operations=[
-                migrations.CreateModel(
-                    name="APIToken",
-                    fields=[
-                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
-                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
-                        ("modified_at", models.DateTimeField(auto_now=True)),
-                        ("token", models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
-                        ("expires", models.DateTimeField(blank=True, null=True)),
-                        (
-                            "created_by",
-                            models.ForeignKey(
-                                default=get_or_create_system_user_pk,
-                                on_delete=django.db.models.deletion.CASCADE,
-                                to=settings.AUTH_USER_MODEL,
-                            ),
-                        ),
-                    ],
-                    options={
-                        "verbose_name": "API Key",
-                        "verbose_name_plural": "API Keys",
-                        "app_label": "api",
-                    },
-                ),
-                migrations.CreateModel(
-                    name="OutboundWebhook",
-                    fields=[
-                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
-                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
-                        ("modified_at", models.DateTimeField(auto_now=True)),
-                        (
-                            "name",
-                            models.CharField(db_index=True, help_text="Webhook name.", max_length=255, unique=True, verbose_name="name"),
-                        ),
-                        (
-                            "signal",
-                            models.CharField(
-                                choices=[
-                                    ("CREATE", "Create"),
-                                    ("UPDATE", "Update"),
-                                    ("DELETE", "Delete"),
-                                    ("M2M", "M2M changed"),
-                                    ("CREATE_OR_UPDATE", "Create or Update"),
-                                    ("CREATE_OR_DELETE", "Create or Delete"),
-                                    ("CREATE_OR_M2M", "Create or M2M changed"),
-                                    ("UPDATE_OR_DELETE", "Update or Delete"),
-                                    ("UPDATE_OR_M2M", "Update or M2M changed"),
-                                    ("DELETE_OR_M2M", "Delete or M2M changed"),
-                                    ("CREATE_UPDATE_OR_DELETE", "Create, Update or Delete"),
-                                    ("CREATE_UPDATE_OR_M2M", "Create, Update or M2M changed"),
-                                    ("CREATE_DELETE_OR_M2M", "Create, Delete or M2M changed"),
-                                    ("UPDATE_DELETE_OR_M2M", "Update, Delete or M2M changed"),
-                                    ("CREATE_UPDATE_DELETE_OR_M2M", "Create, Update or Delete, or M2M changed"),
-                                ],
-                                help_text="Signal the webhook fires to.",
-                                max_length=255,
-                                verbose_name="signal",
-                            ),
-                        ),
-                        (
-                            "ref",
-                            models.CharField(
-                                db_index=True,
-                                help_text="Dot import notation to the model the webhook is for.",
-                                max_length=1023,
-                                validators=[signal_webhooks.utils.model_from_reference],
-                                verbose_name="referenced model",
-                            ),
-                        ),
-                        (
-                            "endpoint",
-                            models.URLField(help_text="Target endpoint for this webhook.", max_length=2047, verbose_name="endpoint"),
-                        ),
-                        (
-                            "headers",
-                            models.JSONField(
-                                blank=True,
-                                default=dict,
-                                help_text="Headers to send with the webhook request.",
-                                validators=[signal_webhooks.utils.is_dict],
-                                verbose_name="headers",
-                            ),
-                        ),
-                        (
-                            "auth_token",
-                            signal_webhooks.fields.TokenField(
-                                blank=True,
-                                default="",
-                                help_text="Authentication token to use in an Authorization header.",
-                                max_length=8000,
-                                validators=[signal_webhooks.utils.decode_cipher_key],
-                                verbose_name="authentication token",
-                            ),
-                        ),
-                        ("enabled", models.BooleanField(default=True, help_text="Is this webhook enabled?", verbose_name="enabled")),
-                        (
-                            "keep_last_response",
-                            models.BooleanField(
-                                default=False,
-                                help_text="Should the webhook keep a log of the latest response it got?",
-                                verbose_name="keep last response",
-                            ),
-                        ),
-                        (
-                            "created",
-                            models.DateTimeField(auto_now_add=True, help_text="When the webhook was created.", verbose_name="created"),
-                        ),
-                        (
-                            "updated",
-                            models.DateTimeField(auto_now=True, help_text="When the webhook was last updated.", verbose_name="updated"),
-                        ),
-                        (
-                            "last_response",
-                            models.CharField(
-                                blank=True,
-                                default="",
-                                help_text="Latest response to this webhook.",
-                                max_length=8000,
-                                verbose_name="last response",
-                            ),
-                        ),
-                        (
-                            "last_success",
-                            models.DateTimeField(
-                                default=None,
-                                help_text="When the webhook last succeeded.",
-                                null=True,
-                                verbose_name="last success",
-                            ),
-                        ),
-                        (
-                            "last_failure",
-                            models.DateTimeField(
-                                default=None,
-                                help_text="When the webhook last failed.",
-                                null=True,
-                                verbose_name="last failure",
-                            ),
-                        ),
-                        (
-                            "created_by",
-                            models.ForeignKey(
-                                default=get_or_create_system_user_pk,
-                                on_delete=django.db.models.deletion.CASCADE,
-                                to=settings.AUTH_USER_MODEL,
-                            ),
-                        ),
-                    ],
-                    options={
-                        "verbose_name": "API Outbound Webhook",
-                        "app_label": "api",
-                    },
-                ),
-                migrations.AddConstraint(
-                    model_name="outboundwebhook",
-                    constraint=models.UniqueConstraint(fields=["ref", "endpoint"], name="prevent_duplicate_hooks_api_outboundwebhook"),
-                ),
+        migrations.CreateModel(
+            name='APIToken',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('token', models.CharField(default=api.models.generate_secret_token, max_length=32, unique=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('expires', models.DateTimeField(blank=True, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
            ],
        ),
    ]
@@ -0,0 +1,17 @@
+# Generated by Django 5.0.4 on 2024-04-26 05:28
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='apitoken',
+            options={'verbose_name': 'API Key', 'verbose_name_plural': 'API Keys'},
+        ),
+    ]
@@ -0,0 +1,77 @@
+# Generated by Django 5.0.6 on 2024-06-03 01:52
+
+import abid_utils.models
+import charidfield.fields
+import django.db.models.deletion
+import signal_webhooks.fields
+import signal_webhooks.utils
+import uuid
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0002_alter_apitoken_options'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='apitoken',
+            old_name='user',
+            new_name='created_by',
+        ),
+        migrations.AddField(
+            model_name='apitoken',
+            name='abid',
+            field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='apt_', unique=True),
+        ),
+        migrations.AddField(
+            model_name='apitoken',
+            name='modified',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name='apitoken',
+            name='uuid',
+            field=models.UUIDField(blank=True, null=True, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False),
+        ),
+        migrations.CreateModel(
+            name='OutboundWebhook',
+            fields=[
+                ('name', models.CharField(db_index=True, help_text='Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).', max_length=255, unique=True, verbose_name='name')),
+                ('signal', models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='The type of event the webhook should fire for (e.g. Create, Update, Delete).', max_length=255, verbose_name='signal')),
+                ('ref', models.CharField(db_index=True, help_text='Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model')),
+                ('endpoint', models.URLField(help_text='External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).', max_length=2047, verbose_name='endpoint')),
+                ('headers', models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers')),
+                ('auth_token', signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token')),
+                ('enabled', models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled')),
+                ('keep_last_response', models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response')),
+                ('updated', models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated')),
+                ('last_response', models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response')),
+                ('last_success', models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success')),
+                ('last_failure', models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('modified', models.DateTimeField(auto_now=True)),
+                ('id', models.UUIDField(blank=True, null=True, unique=True)),
+                ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='whk_', unique=True)),
+                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'verbose_name': 'API Outbound Webhook',
+                'abstract': False,
+            },
+        ),
+        migrations.AddConstraint(
+            model_name='outboundwebhook',
+            constraint=models.UniqueConstraint(fields=('ref', 'endpoint'), name='prevent_duplicate_hooks_api_outboundwebhook'),
+        ),
+    ]
@@ -0,0 +1,24 @@
+# Generated by Django 5.1 on 2024-08-20 10:44
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0003_rename_user_apitoken_created_by_apitoken_abid_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='uuid',
+            field=models.UUIDField(blank=True, editable=False, null=True, unique=True),
+        ),
+    ]
@@ -0,0 +1,22 @@
+# Generated by Django 5.1 on 2024-08-20 22:40
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0004_alter_apitoken_id_alter_apitoken_uuid'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='apitoken',
+            name='uuid',
+        ),
+        migrations.RemoveField(
+            model_name='outboundwebhook',
+            name='id',
+        ),
+    ]
@@ -0,0 +1,29 @@
+# Generated by Django 5.1 on 2024-08-20 22:43
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='outboundwebhook',
+            old_name='uuid',
+            new_name='id'
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+    ]
@@ -0,0 +1,22 @@
+# Generated by Django 5.1 on 2024-08-20 22:52
+
+import abid_utils.models
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0006_remove_outboundwebhook_uuid_apitoken_id_and_more'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='apitoken',
+            name='created_by',
+            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+    ]
@@ -1,54 +1,122 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

+import uuid
 import secrets
-from archivebox.uuid_compat import uuid7
+from datetime import timedelta

 from django.conf import settings
 from django.db import models
 from django.utils import timezone
-from django_stubs_ext.db.models import TypedModelMeta
+
 from signal_webhooks.models import WebhookBase

-from archivebox.base_models.models import get_or_create_system_user_pk
+from django_stubs_ext.db.models import TypedModelMeta
+
+from abid_utils.models import ABIDModel, ABIDField, get_or_create_system_user_pk
+


 def generate_secret_token() -> str:
+    # returns cryptographically secure string with len() == 32
    return secrets.token_hex(16)


-class APIToken(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
+class APIToken(ABIDModel):
+    """
+    A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox.
+    """
+    # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
+    abid_prefix = 'apt_'
+    abid_ts_src = 'self.created'
+    abid_uri_src = 'self.token'
+    abid_subtype_src = 'self.created_by_id'
+    abid_rand_src = 'self.id'
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    created = models.DateTimeField(auto_now_add=True)
+
    token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
    expires = models.DateTimeField(null=True, blank=True)

    class Meta(TypedModelMeta):
-        app_label = "api"
        verbose_name = "API Key"
        verbose_name_plural = "API Keys"

    def __str__(self) -> str:
        return self.token

+    def __repr__(self) -> str:
+        return f'<APIToken user={self.created_by.username} token={self.token_redacted}>'
+
+    def __json__(self) -> dict:
+        return {
+            "TYPE":             "APIToken",    
+            "id":               str(self.pk),
+            "abid":             str(self.ABID),
+            "created_by_id":    str(self.created_by_id),
+            "token":            self.token,
+            "created":          self.created.isoformat(),
+            "expires":          self.expires_as_iso8601,
+        }
+
+    @property
+    def expires_as_iso8601(self):
+        """Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
+        expiry_date = self.expires or (timezone.now() + timedelta(days=365 * 100))
+
+        return expiry_date.isoformat()
+    
    @property
    def token_redacted(self):
-        return f"************{self.token[-4:]}"
+        return f'************{self.token[-4:]}'

    def is_valid(self, for_date=None):
-        return not self.expires or self.expires >= (for_date or timezone.now())
+        for_date = for_date or timezone.now()
+
+        if self.expires and self.expires < for_date:
+            return False
+
+        return True


-class OutboundWebhook(WebhookBase):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
+
+
+
+
+# monkey patch django-signals-webhooks to change how it shows up in Admin UI
+
+class OutboundWebhook(ABIDModel, WebhookBase):
+    """
+    Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using:
+        settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
+    """
+    abid_prefix = 'whk_'
+    abid_ts_src = 'self.created'
+    abid_uri_src = 'self.endpoint'
+    abid_subtype_src = 'self.ref'
+    abid_rand_src = 'self.id'
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+
+    # More fields here: WebhookBase...
+
+    WebhookBase._meta.get_field('name').help_text = (
+        'Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).')
+    WebhookBase._meta.get_field('signal').help_text = (
+        'The type of event the webhook should fire for (e.g. Create, Update, Delete).')
+    WebhookBase._meta.get_field('ref').help_text = (
+        'Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).')
+    WebhookBase._meta.get_field('endpoint').help_text = (
+        'External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).')

    class Meta(WebhookBase.Meta):
-        app_label = "api"
-        verbose_name = "API Outbound Webhook"
+        verbose_name = 'API Outbound Webhook'

-    def __str__(self) -> str:
-        return f"[{self.id}] {self.ref} -> {self.endpoint}"
@@ -0,0 +1,30 @@
+__package__ = 'archivebox.api'
+
+from django.test import TestCase
+from ninja.testing import TestClient
+
+from .routes_cli import router
+
+class ArchiveBoxCLIAPITestCase(TestCase):
+    def setUp(self):
+        self.client = TestClient(router)
+
+    def test_add_endpoint(self):
+        response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])
+
+    def test_remove_endpoint(self):
+        response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])
+
+    def test_update_endpoint(self):
+        response = self.client.post("/update", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])
+
+    def test_list_all_endpoint(self):
+        response = self.client.post("/list_all", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])
@@ -1,29 +1,16 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from urllib.parse import quote
-
-from django.http import HttpRequest, HttpResponseRedirect
-from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView

-from archivebox.core.host_utils import build_web_url
-
 from .v1_api import urls as v1_api_urls

-
-def archive_redirect_view(request: HttpRequest, url: str) -> HttpResponseRedirect:
-    if request.META.get("QUERY_STRING"):
-        url = f"{url}?{request.META['QUERY_STRING']}"
-    return redirect(build_web_url(f"/web/{quote(url, safe=':/')}", request=request))
-
-
 urlpatterns = [
-    path("", RedirectView.as_view(url="/api/v1/docs")),
-    path("archive/<path:url>", archive_redirect_view, name="api-archive-redirect"),
-    path("v1/", RedirectView.as_view(url="/api/v1/docs")),
-    path("v1/", v1_api_urls),
-    path("v1", RedirectView.as_view(url="/api/v1/docs")),
+    path("",                 RedirectView.as_view(url='/api/v1')),
+
+    path("v1/",              v1_api_urls),
+    path("v1",               RedirectView.as_view(url='/api/v1/docs')),
+
    # ... v2 can be added here ...
    # path("v2/",              v2_api_urls),
    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
@@ -1,4 +1,4 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'


 from io import StringIO
@@ -7,22 +7,18 @@ from contextlib import redirect_stdout, redirect_stderr

 from django.http import HttpRequest, HttpResponse
 from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
-from django.contrib.auth.models import User

 from ninja import NinjaAPI, Swagger

 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/

-from archivebox.config import VERSION
-from archivebox.config.version import get_COMMIT_HASH
-
-from archivebox.api.auth import API_AUTH_METHODS
-from archivebox.api.models import APIToken
+from api.auth import API_AUTH_METHODS
+from ..config import VERSION, COMMIT_HASH


-COMMIT_HASH = get_COMMIT_HASH() or "unknown"
+COMMIT_HASH = COMMIT_HASH or 'unknown'

-html_description = f"""
+html_description=f'''
 <h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
 <br/>
 <i><b>WARNING: This API is still in an early development stage and may change!</b></i>
@@ -35,59 +31,37 @@ html_description = f"""
 <li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
 </ul>
 <small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
-"""
+'''


 def register_urls(api: NinjaAPI) -> NinjaAPI:
-    api.add_router("/auth/", "archivebox.api.v1_auth.router")
-    api.add_router("/core/", "archivebox.api.v1_core.router")
-    api.add_router("/crawls/", "archivebox.api.v1_crawls.router")
-    api.add_router("/cli/", "archivebox.api.v1_cli.router")
-    api.add_router("/machine/", "archivebox.api.v1_machine.router")
-    api.add_router("/personas/", "archivebox.api.v1_personas.router")
+    api.add_router('/auth/',     'api.v1_auth.router')
+    api.add_router('/core/',     'api.v1_core.router')
+    api.add_router('/cli/',      'api.v1_cli.router')
    return api


-class NinjaAPIWithIOCapture(NinjaAPI):
+class NinjaAPIWithIOCapture(NinjaAPI):    
    def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
        stdout, stderr = StringIO(), StringIO()

        with redirect_stderr(stderr):
            with redirect_stdout(stdout):
-                setattr(request, "stdout", stdout)
-                setattr(request, "stderr", stderr)
+                request.stdout = stdout
+                request.stderr = stderr

                response = super().create_temporal_response(request)

-        # Disable caching of API responses entirely
-        response["Cache-Control"] = "no-store"
-
-        # Add debug stdout and stderr headers to response
-        response["X-ArchiveBox-Stdout"] = stdout.getvalue().replace("\n", "\\n")[:200]
-        response["X-ArchiveBox-Stderr"] = stderr.getvalue().replace("\n", "\\n")[:200]
-        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
-
-        # Add Auth Headers to response
-        api_token_attr = getattr(request, "_api_token", None)
-        api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
-        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
-
-        response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
-        response["X-ArchiveBox-Auth-Expires"] = token_expiry
-        response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
-        response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
-
-        # import ipdb; ipdb.set_trace()
-        # print('RESPONDING NOW', response)
+        print('RESPONDING NOW', response)

        return response


 api = NinjaAPIWithIOCapture(
-    title="ArchiveBox API",
+    title='ArchiveBox API',
    description=html_description,
-    version=VERSION,
+    version='1.0.0',
+    csrf=False,
    auth=API_AUTH_METHODS,
    urls_namespace="api-1",
    docs=Swagger(settings={"persistAuthorization": True}),
@@ -104,15 +78,15 @@ def generic_exception_handler(request, err):
    if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
        status = 404

-    print("".join(format_exception(err)))
+    print(''.join(format_exception(err)))

    return api.create_response(
        request,
        {
            "succeeded": False,
-            "message": f"{err.__class__.__name__}: {err}",
+            "message": f'{err.__class__.__name__}: {err}',
            "errors": [
-                "".join(format_exception(err)),
+                ''.join(format_exception(err)),
                # or send simpler parent-only traceback:
                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
            ],
@@ -121,6 +95,7 @@ def generic_exception_handler(request, err):
    )


+
 # import orjson
 # from ninja.renderers import BaseRenderer
 # class ORJSONRenderer(BaseRenderer):
@@ -1,65 +1,52 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from django.http import HttpRequest
+from typing import Optional

 from ninja import Router, Schema

-from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token
+from api.models import APIToken
+from api.auth import auth_using_token, auth_using_password


-router = Router(tags=["Authentication"], auth=None)
+router = Router(tags=['Authentication'])


 class PasswordAuthSchema(Schema):
    """Schema for a /get_api_token request"""
-
-    username: str | None = None
-    password: str | None = None
+    username: Optional[str] = None
+    password: Optional[str] = None


-@router.post(
-    "/get_api_token",
-    auth=None,
-    summary="Generate an API token for a given username & password (or currently logged-in user)",
-)  # auth=None because they are not authed yet
-def get_api_token(request: HttpRequest, auth_data: PasswordAuthSchema):
+@router.post("/get_api_token", auth=None, summary='Generate an API token for a given username & password (or currently logged-in user)')             # auth=None because they are not authed yet
+def get_api_token(request, auth_data: PasswordAuthSchema):
    user = auth_using_password(
        username=auth_data.username,
        password=auth_data.password,
        request=request,
    )

-    if user and user.is_superuser:
-        api_token = get_or_create_api_token(user)
-        assert api_token is not None, "Failed to create API token"
-        return {
-            "success": True,
-            "user_id": str(user.pk),
-            "username": user.username,
-            "token": api_token.token,
-            "expires": api_token.expires.isoformat() if api_token.expires else None,
-        }
+    if user:
+        # TODO: support multiple tokens in the future, for now we just have one per user
+        api_token, created = APIToken.objects.get_or_create(created_by_id=user.pk)

+        return api_token.__json__()
+    
    return {"success": False, "errors": ["Invalid credentials"]}


+
 class TokenAuthSchema(Schema):
    """Schema for a /check_api_token request"""
-
    token: str


-@router.post(
-    "/check_api_token",
-    auth=None,
-    summary="Validate an API token to make sure its valid and non-expired",
-)  # auth=None because they are not authed yet
-def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
+@router.post("/check_api_token", auth=None, summary='Validate an API token to make sure its valid and non-expired')        # auth=None because they are not authed yet
+def check_api_token(request, token_data: TokenAuthSchema):
    user = auth_using_token(
        token=token_data.token,
        request=request,
    )
    if user:
        return {"success": True, "user_id": str(user.pk)}
-
+    
    return {"success": False, "user_id": None}
@@ -1,192 +1,167 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-import json
-from io import StringIO
-from typing import Any
+from typing import List, Dict, Any, Optional
 from enum import Enum

-from django.http import HttpRequest
-
 from ninja import Router, Schema
-from pydantic import Field

-from archivebox.misc.util import ansi_to_html
-from archivebox.config.common import get_config
+from ..main import (
+    add,
+    remove,
+    update,
+    list_all,
+    schedule,
+)
+from ..util import ansi_to_html
+from ..config import ONLY_NEW


-# from .auth import API_AUTH_METHODS
-
 # router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=["ArchiveBox CLI Sub-Commands"])
+router = Router(tags=['ArchiveBox CLI Sub-Commands'])


 # Schemas

-JSONType = list[Any] | dict[str, Any] | bool | int | str | None
-
+JSONType = List[Any] | Dict[str, Any] | bool | int | str | None

 class CLICommandResponseSchema(Schema):
    success: bool
-    errors: list[str]
+    errors: List[str]
    result: JSONType
-    result_format: str = "str"
    stdout: str
    stderr: str

-
 class FilterTypeChoices(str, Enum):
-    exact = "exact"
-    substring = "substring"
-    regex = "regex"
-    domain = "domain"
-    tag = "tag"
-    timestamp = "timestamp"
-
+    exact = 'exact'
+    substring = 'substring'
+    regex = 'regex'
+    domain = 'domain'
+    tag = 'tag'
+    timestamp = 'timestamp'

 class StatusChoices(str, Enum):
-    indexed = "indexed"
-    archived = "archived"
-    unarchived = "unarchived"
-    present = "present"
-    valid = "valid"
-    invalid = "invalid"
-    duplicate = "duplicate"
-    orphaned = "orphaned"
-    corrupted = "corrupted"
-    unrecognized = "unrecognized"
+    indexed = 'indexed'
+    archived = 'archived'
+    unarchived = 'unarchived'
+    present = 'present'
+    valid = 'valid'
+    invalid = 'invalid'
+    duplicate = 'duplicate'
+    orphaned = 'orphaned'
+    corrupted = 'corrupted'
+    unrecognized = 'unrecognized'


 class AddCommandSchema(Schema):
-    urls: list[str]
-    snapshot_ids: list[str] | None = None
+    urls: List[str]
    tag: str = ""
    depth: int = 0
-    parser: str = "auto"
-    plugins: str = ""
-    update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
-    overwrite: bool = False
+    update: bool = not ONLY_NEW  # Default to the opposite of ONLY_NEW
+    update_all: bool = False
    index_only: bool = False
-
+    overwrite: bool = False
+    init: bool = False
+    extractors: str = ""
+    parser: str = "auto"

 class UpdateCommandSchema(Schema):
-    resume: str | None = None
-    after: float | None = 0
-    before: float | None = 999999999999999
-    filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = ["https://example.com"]
-    batch_size: int = 100
-    continuous: bool = False
-
+    resume: Optional[float] = 0
+    only_new: bool = ONLY_NEW
+    index_only: bool = False
+    overwrite: bool = False
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
+    status: Optional[StatusChoices] = StatusChoices.unarchived
+    filter_type: Optional[str] = FilterTypeChoices.substring
+    filter_patterns: Optional[List[str]] = ['https://example.com']
+    extractors: Optional[str] = ""

 class ScheduleCommandSchema(Schema):
-    import_path: str | None = None
+    import_path: Optional[str] = None
    add: bool = False
-    show: bool = False
-    foreground: bool = False
-    run_all: bool = False
-    quiet: bool = False
-    every: str | None = None
-    tag: str = ""
+    every: Optional[str] = None
+    tag: str = ''
    depth: int = 0
    overwrite: bool = False
-    update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
+    update: bool = not ONLY_NEW
    clear: bool = False

-
 class ListCommandSchema(Schema):
-    filter_patterns: list[str] | None = ["https://example.com"]
+    filter_patterns: Optional[List[str]] = ['https://example.com']
    filter_type: str = FilterTypeChoices.substring
-    status: StatusChoices = StatusChoices.indexed
-    after: float | None = 0
-    before: float | None = 999999999999999
-    sort: str = "bookmarked_at"
+    status: Optional[StatusChoices] = StatusChoices.indexed
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
+    sort: str = 'added'
    as_json: bool = True
    as_html: bool = False
-    as_csv: str | None = "timestamp,url"
+    as_csv: str | bool = 'timestamp,url'
    with_headers: bool = False

-
 class RemoveCommandSchema(Schema):
    delete: bool = True
-    after: float | None = 0
-    before: float | None = 999999999999999
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
    filter_type: str = FilterTypeChoices.exact
-    filter_patterns: list[str] | None = ["https://example.com"]
+    filter_patterns: Optional[List[str]] = ['https://example.com']


-@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
-def cli_add(request: HttpRequest, args: AddCommandSchema):
-    from archivebox.cli.archivebox_add import add

-    crawl, snapshots = add(
+
+
+@router.post("/add", response=CLICommandResponseSchema, summary='archivebox add [args] [urls]')
+def cli_add(request, args: AddCommandSchema):
+    result = add(
        urls=args.urls,
-        snapshot_ids=args.snapshot_ids,
        tag=args.tag,
        depth=args.depth,
        update=args.update,
+        update_all=args.update_all,
        index_only=args.index_only,
        overwrite=args.overwrite,
-        plugins=args.plugins,
+        init=args.init,
+        extractors=args.extractors,
        parser=args.parser,
-        bg=True,  # Always run in background for API calls
-        created_by_id=request.user.pk,
    )

-    snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
-    result_payload = {
-        "crawl_id": str(crawl.id),
-        "num_snapshots": len(snapshot_ids),
-        "snapshot_ids": snapshot_ids,
-        "queued_urls": args.urls,
-    }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
-
-    return {
-        "success": True,
-        "errors": [],
-        "result": result_payload,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
-def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import update
-
-    result = update(
-        filter_patterns=args.filter_patterns or [],
-        filter_type=args.filter_type or FilterTypeChoices.substring,
-        after=args.after,
-        before=args.before,
-        resume=args.resume,
-        batch_size=args.batch_size,
-        continuous=args.continuous,
-    )
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
    return {
        "success": True,
        "errors": [],
        "result": result,
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
    }


-@router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
-def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
-    from archivebox.cli.archivebox_schedule import schedule
+@router.post("/update", response=CLICommandResponseSchema, summary='archivebox update [args] [filter_patterns]')
+def cli_update(request, args: UpdateCommandSchema):
+    result = update(
+        resume=args.resume,
+        only_new=args.only_new,
+        index_only=args.index_only,
+        overwrite=args.overwrite,
+        before=args.before,
+        after=args.after,
+        status=args.status,
+        filter_type=args.filter_type,
+        filter_patterns=args.filter_patterns,
+        extractors=args.extractors,
+    )
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }

+
+@router.post("/schedule", response=CLICommandResponseSchema, summary='archivebox schedule [args] [import_path]')
+def cli_schedule(request, args: ScheduleCommandSchema):
    result = schedule(
        import_path=args.import_path,
        add=args.add,
        show=args.show,
-        foreground=args.foreground,
-        run_all=args.run_all,
-        quiet=args.quiet,
        clear=args.clear,
        every=args.every,
        tag=args.tag,
@@ -195,23 +170,19 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
        update=args.update,
    )

-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
    return {
        "success": True,
        "errors": [],
        "result": result,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
    }


-@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
-def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_search import search

-    result = search(
+@router.post("/list", response=CLICommandResponseSchema, summary='archivebox list [args] [filter_patterns]')
+def cli_list(request, args: ListCommandSchema):
+    result = list_all(
        filter_patterns=args.filter_patterns,
        filter_type=args.filter_type,
        status=args.status,
@@ -224,64 +195,40 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
        with_headers=args.with_headers,
    )

-    result_format = "txt"
+    result_format = 'txt'
    if args.as_json:
        result_format = "json"
-        result = json.loads(result)
    elif args.as_html:
        result_format = "html"
    elif args.as_csv:
        result_format = "csv"

-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
    return {
        "success": True,
        "errors": [],
        "result": result,
        "result_format": result_format,
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
    }
+    


-@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
-def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
-    from archivebox.cli.archivebox_remove import remove
-    from archivebox.cli.archivebox_search import get_snapshots
-    from archivebox.core.models import Snapshot
-
-    filter_patterns = args.filter_patterns or []
-    snapshots_to_remove = get_snapshots(
-        filter_patterns=filter_patterns,
-        filter_type=args.filter_type,
-        after=args.after,
-        before=args.before,
-    )
-    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
-
-    remove(
-        yes=True,  # no way to interactively ask for confirmation via API, so we force yes
+@router.post("/remove", response=CLICommandResponseSchema, summary='archivebox remove [args] [filter_patterns]')
+def cli_remove(request, args: RemoveCommandSchema):
+    result = remove(
+        yes=True,            # no way to interactively ask for confirmation via API, so we force yes
        delete=args.delete,
-        snapshots=snapshots_to_remove,
        before=args.before,
        after=args.after,
        filter_type=args.filter_type,
-        filter_patterns=filter_patterns,
+        filter_patterns=args.filter_patterns,
    )
-
-    result = {
-        "removed_count": len(removed_snapshot_ids),
-        "removed_snapshot_ids": removed_snapshot_ids,
-        "remaining_snapshots": Snapshot.objects.count(),
-    }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
    return {
        "success": True,
        "errors": [],
        "result": result,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
    }
+    
@@ -1,220 +0,0 @@
-__package__ = "archivebox.api"
-
-from uuid import UUID
-from datetime import datetime
-from django.http import HttpRequest
-from django.shortcuts import redirect
-from django.utils import timezone
-
-from django.contrib.auth import get_user_model
-from django.contrib.auth.models import User
-
-from ninja import Router, Schema
-from ninja.errors import HttpError
-
-from archivebox.core.models import Snapshot
-from archivebox.crawls.models import Crawl
-from archivebox.config.common import get_config
-
-from .auth import API_AUTH_METHODS
-
-router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
-
-
-class CrawlSchema(Schema):
-    TYPE: str = "crawls.models.Crawl"
-
-    id: UUID
-
-    modified_at: datetime
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-
-    status: str
-    retry_at: datetime | None
-
-    urls: str
-    max_depth: int
-    max_urls: int
-    crawl_max_size: int
-    snapshot_max_size: int
-    crawl_max_concurrent_snapshots: int
-    tags_str: str
-    config: dict
-
-    # snapshots: List[SnapshotSchema]
-
-    @staticmethod
-    def resolve_crawl_max_concurrent_snapshots(obj):
-        return int((obj.config or {}).get("CRAWL_MAX_CONCURRENT_SNAPSHOTS") or get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS)
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-
-    @staticmethod
-    def resolve_created_by_username(obj):
-        user_model = get_user_model()
-        user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
-        return username if isinstance(username, str) else str(user)
-
-    @staticmethod
-    def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
-            return obj.snapshot_set.all().distinct()
-        return Snapshot.objects.none()
-
-
-class CrawlUpdateSchema(Schema):
-    status: str | None = None
-    retry_at: datetime | None = None
-    tags: list[str] | None = None
-    tags_str: str | None = None
-
-
-class CrawlCreateSchema(Schema):
-    urls: list[str]
-    max_depth: int = 0
-    max_urls: int = 0
-    crawl_max_size: int = 0
-    snapshot_max_size: int = 0
-    crawl_max_concurrent_snapshots: int | None = None
-    tags: list[str] | None = None
-    tags_str: str = ""
-    label: str = ""
-    notes: str = ""
-    config: dict = {}
-
-
-class CrawlDeleteResponseSchema(Schema):
-    success: bool
-    crawl_id: str
-    deleted_count: int
-    deleted_snapshots: int
-
-
-def normalize_tag_list(tags: list[str] | None = None, tags_str: str = "") -> list[str]:
-    if tags is not None:
-        return [tag.strip() for tag in tags if tag and tag.strip()]
-    return [tag.strip() for tag in tags_str.split(",") if tag.strip()]
-
-
-@router.get("/crawls", response=list[CrawlSchema], url_name="get_crawls")
-def get_crawls(request: HttpRequest):
-    return Crawl.objects.all().distinct()
-
-
-@router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
-def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
-    urls = [url.strip() for url in data.urls if url and url.strip()]
-    if not urls:
-        raise HttpError(400, "At least one URL is required")
-    if data.max_depth not in (0, 1, 2, 3, 4):
-        raise HttpError(400, "max_depth must be between 0 and 4")
-    if data.max_urls < 0:
-        raise HttpError(400, "max_urls must be >= 0")
-    if data.crawl_max_size < 0:
-        raise HttpError(400, "crawl_max_size must be >= 0")
-    if data.snapshot_max_size < 0:
-        raise HttpError(400, "snapshot_max_size must be >= 0")
-    crawl_max_concurrent_snapshots = data.crawl_max_concurrent_snapshots
-    if crawl_max_concurrent_snapshots is None:
-        crawl_max_concurrent_snapshots = get_config().CRAWL_MAX_CONCURRENT_SNAPSHOTS
-    if crawl_max_concurrent_snapshots < 1:
-        raise HttpError(400, "crawl_max_concurrent_snapshots must be >= 1")
-
-    tags = normalize_tag_list(data.tags, data.tags_str)
-    config = dict(data.config or {})
-    config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
-    crawl = Crawl.objects.create(
-        urls="\n".join(urls),
-        max_depth=data.max_depth,
-        max_urls=data.max_urls,
-        crawl_max_size=data.crawl_max_size,
-        snapshot_max_size=data.snapshot_max_size,
-        tags_str=",".join(tags),
-        label=data.label,
-        notes=data.notes,
-        config=config,
-        status=Crawl.StatusChoices.QUEUED,
-        retry_at=timezone.now(),
-        created_by=request.user if isinstance(request.user, User) else None,
-    )
-    crawl.create_snapshots_from_urls()
-    return crawl
-
-
-@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
-def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
-    """Get a specific Crawl by id."""
-    setattr(request, "with_snapshots", with_snapshots)
-    setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
-
-    if crawl and as_rss:
-        query = request.GET.copy()
-        query.pop("as_rss", None)
-        query["crawl_id"] = str(crawl.id)
-        return redirect(f"/api/v1/core/snapshots.rss?{query.urlencode()}")
-
-    return crawl
-
-
-@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
-def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
-    """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
-    payload = data.dict(exclude_unset=True)
-    update_fields = ["modified_at"]
-
-    tags = payload.pop("tags", None)
-    tags_str = payload.pop("tags_str", None)
-    if tags is not None or tags_str is not None:
-        crawl.tags_str = ",".join(normalize_tag_list(tags, tags_str or ""))
-        update_fields.append("tags_str")
-
-    if "status" in payload:
-        if payload["status"] not in Crawl.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        crawl.status = payload["status"]
-        if crawl.status == Crawl.StatusChoices.SEALED and "retry_at" not in payload:
-            crawl.retry_at = None
-        update_fields.append("status")
-
-    if "retry_at" in payload:
-        crawl.retry_at = payload["retry_at"]
-        update_fields.append("retry_at")
-
-    if payload.get("status") == Crawl.StatusChoices.SEALED:
-        cancelled_at = timezone.now()
-        crawl.retry_at = None
-        if "retry_at" not in update_fields:
-            update_fields.append("retry_at")
-        crawl.save(update_fields=update_fields)
-        Snapshot.objects.filter(
-            crawl=crawl,
-            status__in=[Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED],
-        ).update(
-            status=Snapshot.StatusChoices.SEALED,
-            retry_at=None,
-            modified_at=cancelled_at,
-        )
-    else:
-        crawl.save(update_fields=update_fields)
-    return crawl
-
-
-@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
-def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = Crawl.objects.get(id__icontains=crawl_id)
-    crawl_id_str = str(crawl.id)
-    snapshot_count = crawl.snapshot_set.count()
-    deleted_count, _ = crawl.delete()
-    return {
-        "success": True,
-        "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
-        "deleted_snapshots": snapshot_count,
-    }
@@ -1,161 +0,0 @@
-__package__ = "archivebox.api"
-
-from uuid import UUID
-from typing import Annotated
-from datetime import datetime
-
-from django.http import HttpRequest
-
-from ninja import FilterLookup, FilterSchema, Query, Router, Schema
-from ninja.pagination import paginate
-
-from archivebox.api.v1_core import CustomPagination
-
-
-router = Router(tags=["Machine and Dependencies"])
-
-
-# ============================================================================
-# Machine Schemas
-# ============================================================================
-
-
-class MachineSchema(Schema):
-    """Schema for Machine model."""
-
-    TYPE: str = "machine.Machine"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    guid: str
-    hostname: str
-    hw_in_docker: bool
-    hw_in_vm: bool
-    hw_manufacturer: str
-    hw_product: str
-    hw_uuid: str
-    os_arch: str
-    os_family: str
-    os_platform: str
-    os_release: str
-    os_kernel: str
-    stats: dict
-    num_uses_succeeded: int
-    num_uses_failed: int
-
-
-class MachineFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup("id__startswith")] = None
-    hostname: Annotated[str | None, FilterLookup("hostname__icontains")] = None
-    os_platform: Annotated[str | None, FilterLookup("os_platform__icontains")] = None
-    os_arch: Annotated[str | None, FilterLookup("os_arch")] = None
-    hw_in_docker: Annotated[bool | None, FilterLookup("hw_in_docker")] = None
-    hw_in_vm: Annotated[bool | None, FilterLookup("hw_in_vm")] = None
-    bin_providers: Annotated[str | None, FilterLookup("bin_providers__icontains")] = None
-
-
-# ============================================================================
-# Binary Schemas
-# ============================================================================
-
-
-class BinarySchema(Schema):
-    """Schema for Binary model."""
-
-    TYPE: str = "machine.Binary"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    machine_id: UUID
-    machine_hostname: str
-    name: str
-    binproviders: str
-    binprovider: str
-    abspath: str
-    version: str
-    sha256: str
-    status: str
-    is_valid: bool
-    num_uses_succeeded: int
-    num_uses_failed: int
-
-    @staticmethod
-    def resolve_machine_hostname(obj) -> str:
-        return obj.machine.hostname
-
-    @staticmethod
-    def resolve_is_valid(obj) -> bool:
-        return obj.is_valid
-
-
-class BinaryFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup("id__startswith")] = None
-    name: Annotated[str | None, FilterLookup("name__icontains")] = None
-    binprovider: Annotated[str | None, FilterLookup("binprovider")] = None
-    status: Annotated[str | None, FilterLookup("status")] = None
-    machine_id: Annotated[str | None, FilterLookup("machine_id__startswith")] = None
-    version: Annotated[str | None, FilterLookup("version__icontains")] = None
-
-
-# ============================================================================
-# Machine Endpoints
-# ============================================================================
-
-
-@router.get("/machines", response=list[MachineSchema], url_name="get_machines")
-@paginate(CustomPagination)
-def get_machines(request: HttpRequest, filters: Query[MachineFilterSchema]):
-    """List all machines."""
-    from archivebox.machine.models import Machine
-
-    return filters.filter(Machine.objects.all()).distinct()
-
-
-@router.get("/machine/current", response=MachineSchema, url_name="get_current_machine")
-def get_current_machine(request: HttpRequest):
-    """Get the current machine."""
-    from archivebox.machine.models import Machine
-
-    return Machine.current()
-
-
-@router.get("/machine/{machine_id}", response=MachineSchema, url_name="get_machine")
-def get_machine(request: HttpRequest, machine_id: str):
-    """Get a specific machine by ID."""
-    from archivebox.machine.models import Machine
-    from django.db.models import Q
-
-    return Machine.objects.get(Q(id__startswith=machine_id) | Q(hostname__iexact=machine_id))
-
-
-# ============================================================================
-
-
-# ============================================================================
-# Binary Endpoints
-# ============================================================================
-
-
-@router.get("/binaries", response=list[BinarySchema], url_name="get_binaries")
-@paginate(CustomPagination)
-def get_binaries(request: HttpRequest, filters: Query[BinaryFilterSchema]):
-    """List all binaries."""
-    from archivebox.machine.models import Binary
-
-    return filters.filter(Binary.objects.all().select_related("machine")).distinct()
-
-
-@router.get("/binary/{binary_id}", response=BinarySchema, url_name="get_binary")
-def get_binary(request: HttpRequest, binary_id: str):
-    """Get a specific binary by ID."""
-    from archivebox.machine.models import Binary
-
-    return Binary.objects.select_related("machine").get(id__startswith=binary_id)
-
-
-@router.get("/binary/by-name/{name}", response=list[BinarySchema], url_name="get_binaries_by_name")
-def get_binaries_by_name(request: HttpRequest, name: str):
-    """Get all binaries with the given name."""
-    from archivebox.machine.models import Binary
-
-    return list(Binary.objects.filter(name__iexact=name).select_related("machine"))
@@ -1,157 +0,0 @@
-__package__ = "archivebox.api"
-
-import json
-from datetime import datetime
-from typing import Any
-from uuid import UUID
-
-from django.db.models import Q
-from django.http import HttpRequest
-from ninja import Router, Schema
-from pydantic import Field
-
-from archivebox.personas.importers import validate_persona_name
-from archivebox.personas.models import Persona
-
-
-router = Router(tags=["Personas"])
-
-
-class PersonaBrowserSettingsSchema(Schema):
-    user_agent: str = ""
-    viewport_size: str = ""
-    viewport_device_scale_factor: float | None = None
-    language: str = ""
-    timezone: str = ""
-    geolocation: dict[str, Any] | None = None
-
-
-class PersonaSyncSchema(Schema):
-    extension_persona_id: str
-    name: str
-    settings: PersonaBrowserSettingsSchema = Field(default_factory=PersonaBrowserSettingsSchema)
-    cookies_txt: str = ""
-    auth_json: dict[str, Any] = Field(default_factory=dict)
-
-
-class PersonaSchema(Schema):
-    TYPE: str = "personas.models.Persona"
-    id: UUID
-    name: str
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-    config: dict[str, Any] | None
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by.pk)
-
-    @staticmethod
-    def resolve_created_by_username(obj) -> str:
-        return obj.created_by.username
-
-
-class PersonaSyncResponseSchema(Schema):
-    success: bool
-    created: bool
-    persona: PersonaSchema
-    cookies_file_written: bool
-    auth_file_written: bool
-
-
-def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrowserSettingsSchema) -> dict[str, Any]:
-    config: dict[str, Any] = {
-        "BROWSER_EXTENSION_PERSONA_ID": extension_persona_id,
-        "BROWSER_EXTENSION_SYNCED_AT": datetime.utcnow().isoformat() + "Z",
-    }
-
-    if settings.user_agent:
-        config.update(
-            {
-                "USER_AGENT": settings.user_agent,
-                "CHROME_USER_AGENT": settings.user_agent,
-                "WGET_USER_AGENT": settings.user_agent,
-                "CURL_USER_AGENT": settings.user_agent,
-            },
-        )
-    if settings.viewport_size:
-        config.update(
-            {
-                "RESOLUTION": settings.viewport_size,
-                "CHROME_RESOLUTION": settings.viewport_size,
-            },
-        )
-    if settings.viewport_device_scale_factor is not None:
-        config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
-    if settings.language:
-        config["BROWSER_LANGUAGE"] = settings.language
-    if settings.timezone:
-        config["BROWSER_TIMEZONE"] = settings.timezone
-    if settings.geolocation:
-        config["BROWSER_GEOLOCATION"] = settings.geolocation
-
-    return config
-
-
-def find_persona(extension_persona_id: str, name: str) -> Persona | None:
-    return (
-        Persona.objects.filter(
-            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
-        )
-        .order_by("created_at")
-        .first()
-    )
-
-
-@router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
-def get_personas(request: HttpRequest):
-    """List personas available on this ArchiveBox server."""
-    return Persona.objects.all().order_by("name")
-
-
-@router.post("/sync", response=PersonaSyncResponseSchema, url_name="sync_persona")
-def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
-    """
-    Create or update a Persona from a browser extension profile export.
-
-    The extension sends browser settings plus portable auth artifacts. The server
-    keeps browser override settings in Persona.config and writes cookies.txt /
-    auth.json into the persona directory for extractors to consume.
-    """
-    name = payload.name.strip()
-    is_valid, error_message = validate_persona_name(name)
-    if not is_valid:
-        raise ValueError(error_message)
-
-    persona = find_persona(payload.extension_persona_id, name)
-    created = persona is None
-    if persona is None:
-        persona = Persona(name=name)
-        if getattr(request.user, "is_authenticated", False):
-            persona.created_by = request.user
-
-    persona.config = {
-        **(persona.config or {}),
-        **browser_settings_to_config(payload.extension_persona_id, payload.settings),
-    }
-    persona.save()
-    persona.ensure_dirs()
-
-    cookies_written = False
-    if payload.cookies_txt.strip():
-        (persona.path / "cookies.txt").write_text(payload.cookies_txt)
-        cookies_written = True
-
-    auth_written = False
-    if payload.auth_json:
-        (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
-        auth_written = True
-
-    return {
-        "success": True,
-        "created": created,
-        "persona": persona,
-        "cookies_file_written": cookies_written,
-        "auth_file_written": auth_written,
-    }
@@ -1,32 +0,0 @@
-__package__ = "archivebox.api"
-
-from typing import Any
-from collections.abc import Callable
-import logging
-
-from django.db import transaction
-from signal_webhooks.handlers import sync_task_handler
-
-
-logger = logging.getLogger(__name__)
-
-
-def warning_error_handler(hook: Any, error: Exception | None) -> None:
-    if error is not None:
-        logger.warning("Outbound webhook %r failed: %s", hook.name, error)
-        return
-
-    logger.warning("Outbound webhook %r returned a non-success response.", hook.name)
-
-
-def transaction_on_commit_task_handler(hook: Callable[..., None], **kwargs: Any) -> None:
-    def run_webhook() -> None:
-        try:
-            sync_task_handler(hook, **kwargs)
-        except Exception:
-            logger.warning("Outbound webhook failed after transaction commit.", exc_info=True)
-
-    try:
-        transaction.on_commit(run_webhook)
-    except Exception:
-        logger.warning("Could not schedule outbound webhook after transaction commit.", exc_info=True)
@@ -1 +0,0 @@
-__package__ = "archivebox.base_models"
@@ -1,743 +0,0 @@
-"""Base admin classes for models using UUIDv7."""
-
-__package__ = "archivebox.base_models"
-
-import json
-from collections.abc import Mapping
-from typing import NotRequired, TypedDict
-
-from django import forms
-from django.contrib import admin
-from django.db import models
-from django.forms.renderers import BaseRenderer
-from django.http import HttpRequest, QueryDict
-from django.utils.safestring import SafeString, mark_safe
-from django_object_actions import DjangoObjectActions
-
-
-class ConfigOption(TypedDict):
-    plugin: str
-    type: str | list[str]
-    default: object
-    description: str
-    enum: NotRequired[list[object]]
-    pattern: NotRequired[str]
-    minimum: NotRequired[int | float]
-    maximum: NotRequired[int | float]
-
-
-class KeyValueWidget(forms.Widget):
-    """
-    A widget that renders JSON dict as editable key-value input fields
-    with + and - buttons to add/remove rows.
-    Includes autocomplete for available config keys from the plugin system.
-    """
-
-    template_name = ""  # We render manually
-
-    class Media:
-        css = {
-            "all": [],
-        }
-        js = []
-
-    def _get_config_options(self) -> dict[str, ConfigOption]:
-        """Get available config options from plugins."""
-        try:
-            from archivebox.config.common import ArchiveBoxConfig
-            from archivebox.hooks import discover_plugin_configs
-
-            options: dict[str, ConfigOption] = {}
-            skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "CRAWL_OUTPUT_DIR", "SNAP_DIR"}
-            for key, field in ArchiveBoxConfig.model_fields.items():
-                if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
-                    continue
-                default = field.default
-                try:
-                    json.dumps(default)
-                except TypeError:
-                    default = str(default)
-                options[key] = {
-                    "plugin": "archivebox",
-                    "type": str(field.annotation),
-                    "default": default,
-                    "description": field.description or "",
-                }
-
-            plugin_configs = discover_plugin_configs()
-            for plugin_name, schema in plugin_configs.items():
-                for key, prop in schema.get("properties", {}).items():
-                    option: ConfigOption = {
-                        "plugin": plugin_name,
-                        "type": prop.get("type", "string"),
-                        "default": prop.get("default", ""),
-                        "description": prop.get("description", ""),
-                    }
-                    for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in prop:
-                            option[schema_key] = prop[schema_key]
-                    options[key] = option
-            return options
-        except Exception:
-            return {}
-
-    def _parse_value(self, value: object) -> dict[str, object]:
-        # Parse JSON value to dict
-        if value is None:
-            return {}
-        if isinstance(value, str):
-            try:
-                parsed = json.loads(value) if value else {}
-            except json.JSONDecodeError:
-                return {}
-            return parsed if isinstance(parsed, dict) else {}
-        if isinstance(value, Mapping):
-            return {str(key): item for key, item in value.items()}
-        return {}
-
-    def render(
-        self,
-        name: str,
-        value: object,
-        attrs: Mapping[str, str] | None = None,
-        renderer: BaseRenderer | None = None,
-    ) -> SafeString:
-        data = self._parse_value(value)
-
-        widget_id = attrs.get("id", name) if attrs else name
-        config_options = self._get_config_options()
-
-        # Build datalist options
-        datalist_options = "\n".join(
-            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
-            for key, opt in sorted(config_options.items())
-        )
-
-        # Build config metadata as JSON for JS
-        config_meta_json = json.dumps(config_options)
-
-        html = f'''
-        <div id="{widget_id}_container" class="key-value-editor" style="max-width: 700px;">
-            <datalist id="{widget_id}_keys">
-                {datalist_options}
-            </datalist>
-            <div id="{widget_id}_rows" class="key-value-rows">
-        '''
-
-        # Render existing key-value pairs
-        for key, val in data.items():
-            val_str = json.dumps(val) if not isinstance(val, str) else val
-            html += self._render_row(widget_id, key, val_str)
-
-        # Always add one empty row for new entries
-        html += self._render_row(widget_id, "", "")
-
-        html += f'''
-            </div>
-            <div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
-                <button type="button" onclick="addKeyValueRow_{widget_id}()"
-                        style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
-                    + Add Row
-                </button>
-            </div>
-            <input type="hidden" name="{name}" id="{widget_id}" value="">
-            <script>
-                (function() {{
-                    var configMeta_{widget_id} = {config_meta_json};
-                    var rowCounter_{widget_id} = 0;
-
-                    function stringifyValue_{widget_id}(value) {{
-                        return typeof value === 'string' ? value : JSON.stringify(value);
-                    }}
-
-                    function getTypes_{widget_id}(meta) {{
-                        if (!meta || meta.type === undefined || meta.type === null) {{
-                            return [];
-                        }}
-                        return Array.isArray(meta.type) ? meta.type : [meta.type];
-                    }}
-
-                    function getMetaForKey_{widget_id}(key) {{
-                        if (!key) {{
-                            return null;
-                        }}
-
-                        var explicitMeta = configMeta_{widget_id}[key];
-                        if (explicitMeta) {{
-                            return Object.assign({{ key: key }}, explicitMeta);
-                        }}
-
-                        if (key.endsWith('_BINARY')) {{
-                            return {{
-                                key: key,
-                                plugin: 'custom',
-                                type: 'string',
-                                default: '',
-                                description: 'Path to binary executable',
-                            }};
-                        }}
-
-                        if (isRegexConfigKey_{widget_id}(key)) {{
-                            return {{
-                                key: key,
-                                plugin: 'custom',
-                                type: 'string',
-                                default: '',
-                                description: 'Regex pattern list',
-                            }};
-                        }}
-
-                        return null;
-                    }}
-
-                    function describeMeta_{widget_id}(meta) {{
-                        if (!meta) {{
-                            return '';
-                        }}
-
-                        var details = '';
-                        if (Array.isArray(meta.enum) && meta.enum.length) {{
-                            details = 'Allowed: ' + meta.enum.map(stringifyValue_{widget_id}).join(', ');
-                        }} else {{
-                            var types = getTypes_{widget_id}(meta);
-                            if (types.length) {{
-                                details = 'Expected: ' + types.join(' or ');
-                            }}
-                        }}
-
-                        if (meta.minimum !== undefined || meta.maximum !== undefined) {{
-                            var bounds = [];
-                            if (meta.minimum !== undefined) bounds.push('min ' + meta.minimum);
-                            if (meta.maximum !== undefined) bounds.push('max ' + meta.maximum);
-                            details += (details ? ' ' : '') + '(' + bounds.join(', ') + ')';
-                        }}
-
-                        return [meta.description || '', details].filter(Boolean).join(' ');
-                    }}
-
-                    function getExampleInput_{widget_id}(key, meta) {{
-                        var types = getTypes_{widget_id}(meta);
-                        if (key.endsWith('_BINARY')) {{
-                            return 'Example: wget or /usr/bin/wget';
-                        }}
-                        if (key.endsWith('_ARGS_EXTRA') || key.endsWith('_ARGS')) {{
-                            return 'Example: ["--extra-arg"]';
-                        }}
-                        if (types.includes('array')) {{
-                            return 'Example: ["value"]';
-                        }}
-                        if (types.includes('object')) {{
-                            if (key === 'SAVE_ALLOWLIST' || key === 'SAVE_DENYLIST') {{
-                                return 'Example: {{"^https://example\\\\.com": ["wget"]}}';
-                            }}
-                            return 'Example: {{"key": "value"}}';
-                        }}
-                        return '';
-                    }}
-
-                    function isRegexConfigKey_{widget_id}(key) {{
-                        return key === 'URL_ALLOWLIST' ||
-                            key === 'URL_DENYLIST' ||
-                            key === 'SAVE_ALLOWLIST' ||
-                            key === 'SAVE_DENYLIST' ||
-                            key.endsWith('_PATTERN') ||
-                            key.includes('REGEX');
-                    }}
-
-                    function isSimpleFilterPattern_{widget_id}(pattern) {{
-                        return /^[\\w.*:-]+$/.test(pattern);
-                    }}
-
-                    function validateRegexPattern_{widget_id}(pattern) {{
-                        if (!pattern || isSimpleFilterPattern_{widget_id}(pattern)) {{
-                            return '';
-                        }}
-
-                        try {{
-                            new RegExp(pattern);
-                        }} catch (error) {{
-                            return error && error.message ? error.message : 'Invalid regex';
-                        }}
-                        return '';
-                    }}
-
-                    function validateRegexConfig_{widget_id}(key, raw, typeName) {{
-                        if (typeName === 'object') {{
-                            var parsed;
-                            try {{
-                                parsed = JSON.parse(raw);
-                            }} catch (error) {{
-                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
-                            }}
-                            if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {{
-                                return {{ ok: false, value: parsed, message: 'Must be a JSON object' }};
-                            }}
-                            for (var regexKey in parsed) {{
-                                var objectRegexError = validateRegexPattern_{widget_id}(regexKey);
-                                if (objectRegexError) {{
-                                    return {{ ok: false, value: parsed, message: 'Invalid regex key "' + regexKey + '": ' + objectRegexError }};
-                                }}
-                            }}
-                            return {{ ok: true, value: parsed, message: '' }};
-                        }}
-
-                        var patterns = raw.split(/[\\n,]+/).map(function(pattern) {{
-                            return pattern.trim();
-                        }}).filter(Boolean);
-                        for (var i = 0; i < patterns.length; i++) {{
-                            var regexError = validateRegexPattern_{widget_id}(patterns[i]);
-                            if (regexError) {{
-                                return {{ ok: false, value: raw, message: 'Invalid regex "' + patterns[i] + '": ' + regexError }};
-                            }}
-                        }}
-                        return {{ ok: true, value: raw, message: '' }};
-                    }}
-
-                    function validateBinaryValue_{widget_id}(raw) {{
-                        if (!raw) {{
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        if (/['"`]/.test(raw)) {{
-                            return {{ ok: false, value: raw, message: 'Binary paths cannot contain quotes' }};
-                        }}
-
-                        if (/[;&|<>$(){{}}\\[\\]!]/.test(raw)) {{
-                            return {{ ok: false, value: raw, message: 'Binary paths can only be a binary name or absolute path' }};
-                        }}
-
-                        if (raw.startsWith('/')) {{
-                            if (/^[A-Za-z0-9_./+\\- ]+$/.test(raw)) {{
-                                return {{ ok: true, value: raw, message: '' }};
-                            }}
-                            return {{ ok: false, value: raw, message: 'Absolute paths may only contain path-safe characters' }};
-                        }}
-
-                        if (/^[A-Za-z0-9_.+-]+$/.test(raw)) {{
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        return {{ ok: false, value: raw, message: 'Enter a binary name like wget or an absolute path like /usr/bin/wget' }};
-                    }}
-
-                    function parseValue_{widget_id}(raw) {{
-                        try {{
-                            if (raw === 'true') return true;
-                            if (raw === 'false') return false;
-                            if (raw === 'null') return null;
-                            if (raw !== '' && !isNaN(raw)) return Number(raw);
-                            if ((raw.startsWith('{{') && raw.endsWith('}}')) ||
-                                (raw.startsWith('[') && raw.endsWith(']')) ||
-                                (raw.startsWith('"') && raw.endsWith('"'))) {{
-                                return JSON.parse(raw);
-                            }}
-                        }} catch (error) {{
-                            return raw;
-                        }}
-                        return raw;
-                    }}
-
-                    function sameValue_{widget_id}(left, right) {{
-                        return left === right || JSON.stringify(left) === JSON.stringify(right);
-                    }}
-
-                    function parseTypedValue_{widget_id}(raw, typeName, meta) {{
-                        var numberValue;
-                        var parsed;
-
-                        if (typeName && meta && meta.key && isRegexConfigKey_{widget_id}(meta.key)) {{
-                            return validateRegexConfig_{widget_id}(meta.key, raw, typeName);
-                        }}
-
-                        if (typeName === 'string' && meta && meta.key && meta.key.endsWith('_BINARY')) {{
-                            return validateBinaryValue_{widget_id}(raw);
-                        }}
-
-                        if (typeName === 'string') {{
-                            if (meta.pattern) {{
-                                try {{
-                                    if (!(new RegExp(meta.pattern)).test(raw)) {{
-                                        return {{ ok: false, value: raw, message: 'Must match pattern ' + meta.pattern }};
-                                    }}
-                                }} catch (error) {{}}
-                            }}
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        if (typeName === 'integer') {{
-                            if (!/^-?\\d+$/.test(raw)) {{
-                                return {{ ok: false, value: raw, message: 'Must be an integer' }};
-                            }}
-                            numberValue = Number(raw);
-                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
-                            }}
-                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
-                            }}
-                            return {{ ok: true, value: numberValue, message: '' }};
-                        }}
-
-                        if (typeName === 'number') {{
-                            if (raw === '' || isNaN(raw)) {{
-                                return {{ ok: false, value: raw, message: 'Must be a number' }};
-                            }}
-                            numberValue = Number(raw);
-                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
-                            }}
-                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
-                            }}
-                            return {{ ok: true, value: numberValue, message: '' }};
-                        }}
-
-                        if (typeName === 'boolean') {{
-                            var lowered = raw.toLowerCase();
-                            if (lowered === 'true' || raw === '1') return {{ ok: true, value: true, message: '' }};
-                            if (lowered === 'false' || raw === '0') return {{ ok: true, value: false, message: '' }};
-                            return {{ ok: false, value: raw, message: 'Must be true or false' }};
-                        }}
-
-                        if (typeName === 'null') {{
-                            return raw === 'null'
-                                ? {{ ok: true, value: null, message: '' }}
-                                : {{ ok: false, value: raw, message: 'Must be null' }};
-                        }}
-
-                        if (typeName === 'array' || typeName === 'object') {{
-                            try {{
-                                parsed = JSON.parse(raw);
-                            }} catch (error) {{
-                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
-                            }}
-
-                            if (typeName === 'array' && Array.isArray(parsed)) {{
-                                return {{ ok: true, value: parsed, message: '' }};
-                            }}
-                            if (typeName === 'object' && parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {{
-                                return {{ ok: true, value: parsed, message: '' }};
-                            }}
-
-                            return {{
-                                ok: false,
-                                value: parsed,
-                                message: typeName === 'array' ? 'Must be a JSON array' : 'Must be a JSON object',
-                            }};
-                        }}
-
-                        return {{ ok: true, value: parseValue_{widget_id}(raw), message: '' }};
-                    }}
-
-                    function validateValueAgainstMeta_{widget_id}(raw, meta) {{
-                        if (!meta || raw === '') {{
-                            return {{ state: 'neutral', value: raw, message: '' }};
-                        }}
-
-                        var enumValues = Array.isArray(meta.enum) ? meta.enum : [];
-                        var types = getTypes_{widget_id}(meta);
-                        if (!types.length) {{
-                            types = ['string'];
-                        }}
-
-                        var error = 'Invalid value';
-                        for (var i = 0; i < types.length; i++) {{
-                            var candidate = parseTypedValue_{widget_id}(raw, types[i], meta);
-                            if (!candidate.ok) {{
-                                error = candidate.message || error;
-                                continue;
-                            }}
-                            if (enumValues.length && !enumValues.some(function(enumValue) {{
-                                return sameValue_{widget_id}(enumValue, candidate.value) || stringifyValue_{widget_id}(enumValue) === raw;
-                            }})) {{
-                                error = 'Must be one of: ' + enumValues.map(stringifyValue_{widget_id}).join(', ');
-                                continue;
-                            }}
-                            return {{ state: 'valid', value: candidate.value, message: '' }};
-                        }}
-
-                        return {{ state: 'invalid', value: raw, message: error }};
-                    }}
-
-                    function ensureRowId_{widget_id}(row) {{
-                        if (!row.dataset.rowId) {{
-                            row.dataset.rowId = String(rowCounter_{widget_id}++);
-                        }}
-                        return row.dataset.rowId;
-                    }}
-
-                    function setRowHelp_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var help = row.querySelector('.kv-help');
-                        if (!keyInput || !help) {{
-                            return;
-                        }}
-
-                        var key = keyInput.value.trim();
-                        if (!key) {{
-                            help.textContent = '';
-                            return;
-                        }}
-
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (meta) {{
-                            var extra = isRegexConfigKey_{widget_id}(key)
-                                ? ((meta.type === 'object' || (Array.isArray(meta.type) && meta.type.includes('object')))
-                                    ? ' Expected: JSON object with regex keys.'
-                                    : ' Expected: valid regex.')
-                                : '';
-                            var example = getExampleInput_{widget_id}(key, meta);
-                            help.textContent = [describeMeta_{widget_id}(meta) + extra, example].filter(Boolean).join(' ');
-                        }} else {{
-                            help.textContent = 'Custom key';
-                        }}
-                    }}
-
-                    function configureValueInput_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var valueInput = row.querySelector('.kv-value');
-                        var datalist = row.querySelector('.kv-value-options');
-                        if (!keyInput || !valueInput || !datalist) {{
-                            return;
-                        }}
-
-                        var rowId = ensureRowId_{widget_id}(row);
-                        datalist.id = '{widget_id}_value_options_' + rowId;
-
-                        var meta = getMetaForKey_{widget_id}(keyInput.value.trim());
-                        var enumValues = Array.isArray(meta && meta.enum) ? meta.enum : [];
-                        var types = getTypes_{widget_id}(meta);
-                        if (!enumValues.length && types.includes('boolean')) {{
-                            enumValues = ['True', 'False'];
-                        }}
-                        if (enumValues.length) {{
-                            datalist.innerHTML = enumValues.map(function(enumValue) {{
-                                return '<option value="' + stringifyValue_{widget_id}(enumValue).replace(/"/g, '&quot;') + '"></option>';
-                            }}).join('');
-                            valueInput.setAttribute('list', datalist.id);
-                        }} else {{
-                            datalist.innerHTML = '';
-                            valueInput.removeAttribute('list');
-                        }}
-                    }}
-
-                    function setValueValidationState_{widget_id}(input, state, message) {{
-                        if (!input) {{
-                            return;
-                        }}
-
-                        if (state === 'valid') {{
-                            input.style.borderColor = '#2da44e';
-                            input.style.boxShadow = '0 0 0 1px rgba(45, 164, 78, 0.18)';
-                            input.style.backgroundColor = '#f6ffed';
-                        }} else if (state === 'invalid') {{
-                            input.style.borderColor = '#cf222e';
-                            input.style.boxShadow = '0 0 0 1px rgba(207, 34, 46, 0.18)';
-                            input.style.backgroundColor = '#fff8f8';
-                        }} else {{
-                            input.style.borderColor = '#ccc';
-                            input.style.boxShadow = 'none';
-                            input.style.backgroundColor = '';
-                        }}
-                        input.title = message || '';
-                    }}
-
-                    function applyValueValidation_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var valueInput = row.querySelector('.kv-value');
-                        if (!keyInput || !valueInput) {{
-                            return;
-                        }}
-
-                        var key = keyInput.value.trim();
-                        if (!key) {{
-                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
-                            return;
-                        }}
-
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (!meta) {{
-                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
-                            return;
-                        }}
-
-                        var validation = validateValueAgainstMeta_{widget_id}(valueInput.value.trim(), meta);
-                        setValueValidationState_{widget_id}(valueInput, validation.state, validation.message);
-                    }}
-
-                    function coerceValueForStorage_{widget_id}(key, raw) {{
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (!meta) {{
-                            return parseValue_{widget_id}(raw);
-                        }}
-
-                        var validation = validateValueAgainstMeta_{widget_id}(raw, meta);
-                        return validation.state === 'valid' ? validation.value : raw;
-                    }}
-
-                    function initializeRows_{widget_id}() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
-                            ensureRowId_{widget_id}(row);
-                            configureValueInput_{widget_id}(row);
-                            setRowHelp_{widget_id}(row);
-                            applyValueValidation_{widget_id}(row);
-                        }});
-                    }}
-
-                    function updateHiddenField_{widget_id}() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        var rows = container.querySelectorAll('.key-value-row');
-                        var result = {{}};
-                        rows.forEach(function(row) {{
-                            var keyInput = row.querySelector('.kv-key');
-                            var valInput = row.querySelector('.kv-value');
-                            if (keyInput && valInput && keyInput.value.trim()) {{
-                                var key = keyInput.value.trim();
-                                var val = valInput.value.trim();
-                                result[key] = coerceValueForStorage_{widget_id}(key, val);
-                            }}
-                        }});
-                        document.getElementById('{widget_id}').value = JSON.stringify(result);
-                    }}
-
-                    window.addKeyValueRow_{widget_id} = function() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        var newRow = document.createElement('div');
-                        newRow.className = 'key-value-row';
-                        newRow.style.cssText = 'margin-bottom: 6px;';
-                        newRow.innerHTML = '<div style="display: flex; gap: 8px; align-items: center;">' +
-                            '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
-                            'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
-                            '<input type="text" class="kv-value" placeholder="value" ' +
-                            'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
-                            '<datalist class="kv-value-options"></datalist>' +
-                            '<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
-                            'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>' +
-                            '</div>' +
-                            '<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>';
-                        container.appendChild(newRow);
-                        ensureRowId_{widget_id}(newRow);
-                        configureValueInput_{widget_id}(newRow);
-                        setRowHelp_{widget_id}(newRow);
-                        applyValueValidation_{widget_id}(newRow);
-                        updateHiddenField_{widget_id}();
-                        newRow.querySelector('.kv-key').focus();
-                    }};
-
-                    window.removeKeyValueRow_{widget_id} = function(btn) {{
-                        var row = btn.closest('.key-value-row');
-                        row.remove();
-                        updateHiddenField_{widget_id}();
-                    }};
-
-                    window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
-
-                    // Initialize on load
-                    document.addEventListener('DOMContentLoaded', function() {{
-                        initializeRows_{widget_id}();
-                        updateHiddenField_{widget_id}();
-                    }});
-                    // Also run immediately in case DOM is already ready
-                    if (document.readyState !== 'loading') {{
-                        initializeRows_{widget_id}();
-                        updateHiddenField_{widget_id}();
-                    }}
-
-                    // Update on any input change
-                    var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
-
-                    rowsEl_{widget_id}.addEventListener('input', function(event) {{
-                        var row = event.target.closest('.key-value-row');
-                        if (!row) {{
-                            return;
-                        }}
-
-                        if (event.target.classList.contains('kv-key')) {{
-                            configureValueInput_{widget_id}(row);
-                            setRowHelp_{widget_id}(row);
-                        }}
-
-                        if (event.target.classList.contains('kv-key') || event.target.classList.contains('kv-value')) {{
-                            applyValueValidation_{widget_id}(row);
-                            updateHiddenField_{widget_id}();
-                        }}
-                    }});
-                }})();
-            </script>
-        </div>
-        '''
-        return mark_safe(html)
-
-    def _render_row(self, widget_id: str, key: str, value: str) -> str:
-        return f'''
-            <div class="key-value-row" style="margin-bottom: 6px;">
-                <div style="display: flex; gap: 8px; align-items: center;">
-                    <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
-                           style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <input type="text" class="kv-value" value="{self._escape(value)}" placeholder="value"
-                           style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <datalist class="kv-value-options"></datalist>
-                    <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
-                            style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>
-                </div>
-                <div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>
-            </div>
-        '''
-
-    def _escape(self, s: object) -> str:
-        """Escape HTML special chars in attribute values."""
-        if not s:
-            return ""
-        return str(s).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
-
-    def value_from_datadict(
-        self,
-        data: QueryDict | Mapping[str, object],
-        files: object,
-        name: str,
-    ) -> str:
-        value = data.get(name, "{}")
-        return value if isinstance(value, str) else "{}"
-
-
-class ConfigEditorMixin(admin.ModelAdmin):
-    """
-    Mixin for admin classes with a config JSON field.
-
-    Provides a key-value editor widget with autocomplete for available config keys.
-    """
-
-    def formfield_for_dbfield(
-        self,
-        db_field: models.Field[object, object],
-        request: HttpRequest,
-        **kwargs: object,
-    ) -> forms.Field | None:
-        """Use KeyValueWidget for the config JSON field."""
-        if db_field.name == "config":
-            kwargs["widget"] = KeyValueWidget()
-        return super().formfield_for_dbfield(db_field, request, **kwargs)
-
-
-class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
-    list_display = ("id", "created_at", "created_by")
-    readonly_fields = ("id", "created_at", "modified_at")
-    show_search_mode_selector = False
-
-    def get_default_search_mode(self) -> str:
-        # The shared changelist template always asks every admin for a default
-        # search mode, even when the search-mode toggle is hidden.
-        return "meta"
-
-    def get_form(
-        self,
-        request: HttpRequest,
-        obj: models.Model | None = None,
-        change: bool = False,
-        **kwargs: object,
-    ):
-        form = super().get_form(request, obj, change=change, **kwargs)
-        if "created_by" in form.base_fields:
-            form.base_fields["created_by"].initial = request.user
-        return form
@@ -1,7 +0,0 @@
-# from django.apps import AppConfig
-
-
-# class BaseModelsConfig(AppConfig):
-#     default_auto_field = 'django.db.models.BigAutoField'
-
-#     name = 'base_models'
@@ -1,137 +0,0 @@
-"""Base models using UUIDv7 for all id fields."""
-
-__package__ = "archivebox.base_models"
-
-from archivebox.uuid_compat import uuid7
-from pathlib import Path
-
-from django.db import models
-from django.db.models import F
-from django.db import transaction
-from django.utils import timezone
-from django.contrib.auth import get_user_model
-from django.urls import reverse_lazy
-from django.conf import settings
-
-from django_stubs_ext.db.models import TypedModelMeta
-
-
-def get_or_create_system_user_pk(username="system"):
-    User = get_user_model()
-    # If there's exactly one superuser, use that for all system operations
-    if User.objects.filter(is_superuser=True).count() == 1:
-        return User.objects.filter(is_superuser=True).values_list("pk", flat=True)[0]
-    # Otherwise get or create the system user
-    user, _ = User.objects.get_or_create(
-        username=username,
-        defaults={"is_staff": True, "is_superuser": True, "email": "", "password": "!"},
-    )
-    return user.pk
-
-
-class AutoDateTimeField(models.DateTimeField):
-    """DateTimeField that automatically updates on save (legacy compatibility)."""
-
-    def pre_save(self, model_instance, add):
-        if add or not getattr(model_instance, self.attname):
-            value = timezone.now()
-            setattr(model_instance, self.attname, value)
-            return value
-        return super().pre_save(model_instance, add)
-
-
-class ModelWithUUID(models.Model):
-    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-    created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL,
-        on_delete=models.CASCADE,
-        default=get_or_create_system_user_pk,
-        null=False,
-        db_index=True,
-    )
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def __str__(self) -> str:
-        return f"[{self.id}] {self.__class__.__name__}"
-
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
-    @property
-    def api_url(self) -> str:
-        return str(reverse_lazy("api-1:get_any", args=[self.id]))
-
-    @property
-    def api_docs_url(self) -> str:
-        return f"/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}"
-
-
-class ModelWithNotes(models.Model):
-    """Mixin for models with a notes field."""
-
-    notes = models.TextField(blank=True, null=False, default="")
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-
-class ModelWithHealthStats(models.Model):
-    """Mixin for models with health tracking fields."""
-
-    num_uses_failed = models.PositiveIntegerField(default=0)
-    num_uses_succeeded = models.PositiveIntegerField(default=0)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    @property
-    def health(self) -> int:
-        total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
-        return round((self.num_uses_succeeded / total) * 100)
-
-    def increment_health_stats(self, success: bool):
-        """Atomically increment success or failure counter using F() expression."""
-        field = "num_uses_succeeded" if success else "num_uses_failed"
-        type(self).objects.filter(pk=self.pk).update(**{field: F(field) + 1})
-
-
-class ModelWithConfig(models.Model):
-    """Mixin for models with a JSON config field."""
-
-    config = models.JSONField(default=dict, null=True, blank=True, editable=True)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-
-class ModelWithOutputDir(ModelWithUUID):
-    class Meta(ModelWithUUID.Meta):
-        abstract = True
-
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        output_dir = Path(self.output_dir)
-        # Avoid holding SQLite write transactions open across slow filesystem work.
-        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
-        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
-
-    @property
-    def output_dir_parent(self) -> str:
-        return f"{self._meta.model_name}s"
-
-    @property
-    def output_dir_name(self) -> str:
-        return str(self.id)
-
-    @property
-    def output_dir_str(self) -> str:
-        return f"{self.output_dir_parent}/{self.output_dir_name}"
-
-    @property
-    def output_dir(self) -> Path:
-        raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.
@@ -0,0 +1,82 @@
+import sys
+import inspect
+from typing import List, Dict, Any, Optional
+from pathlib import Path
+
+import django
+from django.apps import AppConfig
+from django.core.checks import Tags, Warning, register
+from django.db.backends.sqlite3.base import Database as sqlite3
+
+from pydantic import (
+    Field,
+    SerializeAsAny,
+)
+
+from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
+
+from plugantic.extractors import Extractor, ExtractorName
+from plugantic.plugins import Plugin
+from plugantic.configs import ConfigSet, ConfigSectionName
+from plugantic.replayers import Replayer
+
+
+class PythonBinary(Binary):
+    name: BinName = 'python'
+
+    providers_supported: List[BinProvider] = [EnvProvider()]
+    provider_overrides: Dict[str, Any] = {
+        'env': {
+            'subdeps': \
+                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
+            'abspath': \
+                lambda: sys.executable,
+            'version': \
+                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
+        },
+    }
+
+class SqliteBinary(Binary):
+    name: BinName = 'sqlite'
+    providers_supported: List[BinProvider] = [EnvProvider()]
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'env': {
+            'abspath': \
+                lambda: inspect.getfile(sqlite3),
+            'version': \
+                lambda: sqlite3.version,
+        },
+    }
+
+class DjangoBinary(Binary):
+    name: BinName = 'django'
+
+    providers_supported: List[BinProvider] = [EnvProvider()]
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'env': {
+            'abspath': \
+                lambda: inspect.getfile(django),
+            'version': \
+                lambda: django.VERSION[:3],
+        },
+    }
+
+
+class BasicReplayer(Replayer):
+    name: str = 'basic'
+
+
+class BasePlugin(Plugin):
+    name: str = 'base'
+    configs: List[SerializeAsAny[ConfigSet]] = []
+    binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
+    extractors: List[SerializeAsAny[Extractor]] = []
+    replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()]
+
+
+PLUGINS = [BasePlugin()]
+
+
+class BaseConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'builtin_plugins.base'
@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.
@@ -0,0 +1,94 @@
+from typing import List, Optional
+from pathlib import Path
+
+from django.apps import AppConfig
+from django.core.checks import Tags, Warning, register
+
+from pydantic import (
+    Field,
+    SerializeAsAny,
+)
+
+from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider
+
+from plugantic.extractors import Extractor, ExtractorName
+from plugantic.plugins import Plugin
+from plugantic.configs import ConfigSet, ConfigSectionName
+
+
+###################### Config ##########################
+
+class SinglefileToggleConfig(ConfigSet):
+    section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
+
+    SAVE_SINGLEFILE: bool = True
+
+
+class SinglefileDependencyConfig(ConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    SINGLEFILE_BINARY: str = Field(default='wget')
+    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
+    SINGLEFILE_EXTRA_ARGS: List[str] = []
+    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
+
+class SinglefileOptionsConfig(ConfigSet):
+    section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
+
+    # loaded from shared config
+    SINGLEFILE_USER_AGENT: str = Field(default='', alias='USER_AGENT')
+    SINGLEFILE_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
+    SINGLEFILE_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
+    SINGLEFILE_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
+    SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
+
+
+
+DEFAULT_CONFIG = {
+    'CHECK_SSL_VALIDITY': False,
+    'SAVE_SINGLEFILE': True,
+    'TIMEOUT': 120,
+}
+
+PLUGIN_CONFIG = [
+    SinglefileToggleConfig(**DEFAULT_CONFIG),
+    SinglefileDependencyConfig(**DEFAULT_CONFIG),
+    SinglefileOptionsConfig(**DEFAULT_CONFIG),
+]
+
+###################### Binaries ############################
+
+class SinglefileBinary(Binary):
+    name: BinName = 'single-file'
+    providers_supported: List[BinProvider] = [EnvProvider(), NpmProvider()]
+
+
+###################### Extractors ##########################
+
+class SinglefileExtractor(Extractor):
+    name: ExtractorName = 'singlefile'
+    binary: Binary = SinglefileBinary()
+
+    def get_output_path(self, snapshot) -> Path:
+        return Path(snapshot.link_dir) / 'singlefile.html'
+
+
+###################### Plugins #############################
+
+
+class SinglefilePlugin(Plugin):
+    name: str = 'singlefile'
+    configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG]
+    binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()]
+    extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()]
+
+PLUGINS = [SinglefilePlugin()]
+
+###################### Django Apps #########################
+
+class SinglefileConfig(AppConfig):
+    name = 'builtin_plugins.singlefile'
+    verbose_name = 'SingleFile'
+
+    def ready(self):
+        print('Loaded singlefile plugin')
@@ -0,0 +1,66 @@
+name: singlefile
+plugin_version: '0.0.1'
+plugin_spec: '0.0.1'
+
+binaries:
+    singlefile:
+        providers:
+            - env
+            - npm
+
+commands:
+    - singlefile.exec
+    - singlefile.extract
+    - singlefile.should_extract
+    - singlefile.get_output_path
+
+extractors:
+    singlefile:
+        binary: singlefile
+        test: singlefile.should_extract
+        extract: singlefile.extract
+        output_files:
+            - singlefile.html
+
+configs:
+    ARCHIVE_METHOD_TOGGLES:
+        SAVE_SINGLEFILE:
+            type: bool
+            default: true
+
+    DEPENDENCY_CONFIG:
+        SINGLEFILE_BINARY:
+            type: str
+            default: wget
+        SINGLEFILE_ARGS:
+            type: Optional[List[str]]
+            default: null
+        SINGLEFILE_EXTRA_ARGS:
+            type: List[str]
+            default: []
+        SINGLEFILE_DEFAULT_ARGS:
+            type: List[str]
+            default: 
+            - "--timeout={TIMEOUT-10}"
+
+    ARCHIVE_METHOD_OPTIONS:
+        SINGLEFILE_USER_AGENT:
+            type: str
+            default: ""
+            alias: USER_AGENT
+        SINGLEFILE_TIMEOUT:
+            type: int
+            default: 60
+            alias: TIMEOUT
+        SINGLEFILE_CHECK_SSL_VALIDITY:
+            type: bool
+            default: true
+            alias: CHECK_SSL_VALIDITY
+        SINGLEFILE_RESTRICT_FILE_NAMES:
+            type: str
+            default: windows
+            alias: RESTRICT_FILE_NAMES
+        SINGLEFILE_COOKIES_FILE:
+            type: Optional[Path]
+            default: null
+            alias: COOKIES_FILE
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
@@ -1,199 +1,196 @@
-__package__ = "archivebox.cli"
-__command__ = "archivebox"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox'
+
 import os
 import sys
+import argparse
+import threading
+from time import sleep
+
+from typing import Optional, Dict, List, IO, Union, Iterable
+from pathlib import Path
+
+from ..config import OUTPUT_DIR, check_data_folder, check_migrations, stderr
+
 from importlib import import_module

-import rich_click as click
-from rich import print
+BUILTIN_LIST = list

-from archivebox.config.version import VERSION
+CLI_DIR = Path(__file__).resolve().parent
+
+# these common commands will appear sorted before any others for ease-of-use
+meta_cmds = ('help', 'version')                               # dont require valid data folder at all
+main_cmds = ('init', 'config', 'setup')                       # dont require existing db present
+archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
+fake_db = ("oneshot",)                                        # use fake in-memory db
+
+display_first = (*meta_cmds, *main_cmds, *archive_cmds)
+
+# every imported command module must have these properties in order to be valid
+required_attrs = ('__package__', '__command__', 'main')
+
+# basic checks to make sure imported files are valid subcommands
+is_cli_module = lambda fname: fname.startswith('archivebox_') and fname.endswith('.py')
+is_valid_cli_module = lambda module, subcommand: (
+    all(hasattr(module, attr) for attr in required_attrs)
+    and module.__command__.split(' ')[-1] == subcommand
+)


-if "--debug" in sys.argv:
-    os.environ["DEBUG"] = "True"
-    sys.argv.remove("--debug")
+IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread')  # threads we dont have to wait for before exiting


-class ArchiveBoxGroup(click.Group):
-    """lazy loading click group for archivebox commands"""
+def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: Iterable[str]=IGNORED_BG_THREADS, timeout: int=60) -> int:
+    """
+    Block until the specified threads exit. e.g. pass thread_names=('default_hook_handler',) to wait for webhooks.
+    Useful for waiting for signal handlers, webhooks, etc. to finish running after a mgmt command completes.
+    """

-    meta_commands = {
-        "help": "archivebox.cli.archivebox_help.main",
-        "version": "archivebox.cli.archivebox_version.main",
-        "mcp": "archivebox.cli.archivebox_mcp.main",
-    }
-    setup_commands = {
-        "init": "archivebox.cli.archivebox_init.main",
-        "install": "archivebox.cli.archivebox_install.main",
-    }
-    # Model commands (CRUD operations via subcommands)
-    model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot.main",
-        "archiveresult": "archivebox.cli.archivebox_archiveresult.main",
-        "tag": "archivebox.cli.archivebox_tag.main",
-        "binary": "archivebox.cli.archivebox_binary.main",
-        "process": "archivebox.cli.archivebox_process.main",
-        "machine": "archivebox.cli.archivebox_machine.main",
-        "persona": "archivebox.cli.archivebox_persona.main",
-    }
-    archive_commands = {
-        # High-level commands
-        "add": "archivebox.cli.archivebox_add.main",
-        "extract": "archivebox.cli.archivebox_extract.main",
-        "list": "archivebox.cli.archivebox_list.main",
-        "remove": "archivebox.cli.archivebox_remove.main",
-        "run": "archivebox.cli.archivebox_run.main",
-        "update": "archivebox.cli.archivebox_update.main",
-        "status": "archivebox.cli.archivebox_status.main",
-        "search": "archivebox.cli.archivebox_search.main",
-        "config": "archivebox.cli.archivebox_config.main",
-        "schedule": "archivebox.cli.archivebox_schedule.main",
-        "server": "archivebox.cli.archivebox_server.main",
-        "shell": "archivebox.cli.archivebox_shell.main",
-        "manage": "archivebox.cli.archivebox_manage.main",
-        # Introspection commands
-        "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
-    }
-    legacy_model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
-    }
-    all_subcommands = {
-        **meta_commands,
-        **setup_commands,
-        **model_commands,
-        **archive_commands,
-    }
-    renamed_commands = {
-        "setup": "install",
-        "import": "add",
-        "archive": "add",
-    }
-    legacy_model_subcommands = {
-        "crawl": {"create", "list", "update", "delete"},
-        "snapshot": {"create", "list", "update", "delete"},
-    }
+    wait_for_all: bool = thread_names == ()

-    @classmethod
-    def get_canonical_name(cls, cmd_name):
-        return cls.renamed_commands.get(cmd_name, cmd_name)
+    thread_matches = lambda thread, ptns: any(ptn in repr(thread) for ptn in ptns)

-    @classmethod
-    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
-        if cmd_name not in cls.legacy_model_commands:
-            return False
+    should_wait = lambda thread: (
+        not thread_matches(thread, ignore_names)
+        and (wait_for_all or thread_matches(thread, thread_names)))

-        try:
-            arg_idx = sys.argv.index(cmd_name)
-        except ValueError:
-            return False
+    for tries in range(timeout):
+        all_threads = [*threading.enumerate()]
+        blocking_threads = [*filter(should_wait, all_threads)]
+        threads_summary = ', '.join(repr(t) for t in blocking_threads)
+        if blocking_threads:
+            sleep(1)
+            if tries == 5:                            # only show stderr message if we need to wait more than 5s
+                stderr(
+                    f'[…] Waiting up to {timeout}s for background jobs (e.g. webhooks) to finish...',
+                    threads_summary,
+                )
+        else:
+            return tries

-        remaining_args = sys.argv[arg_idx + 1 :]
-        if not remaining_args:
-            return False
-
-        first_arg = remaining_args[0]
-        if first_arg in ("-h", "--help"):
-            return False
-
-        return first_arg not in cls.legacy_model_subcommands[cmd_name]
-
-    def get_command(self, ctx, cmd_name):
-        # handle renamed commands
-        if cmd_name in self.renamed_commands:
-            new_name = self.renamed_commands[cmd_name]
-            print(
-                f" [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`",
-                file=sys.stderr,
-            )
-            cmd_name = new_name
-            ctx.invoked_subcommand = cmd_name
-
-        if self._should_use_legacy_model_command(cmd_name):
-            return self._lazy_load(self.legacy_model_commands[cmd_name])
-
-        # handle lazy loading of commands
-        if cmd_name in self.all_subcommands:
-            return self._lazy_load(cmd_name)
-
-        # fall-back to using click's default command lookup
-        return super().get_command(ctx, cmd_name)
-
-    @classmethod
-    def _lazy_load(cls, cmd_name_or_path):
-        import_path = cls.all_subcommands.get(cmd_name_or_path)
-        if import_path is None:
-            import_path = cmd_name_or_path
-        modname, funcname = import_path.rsplit(".", 1)
-
-        # print(f'LAZY LOADING {import_path}')
-        mod = import_module(modname)
-        func = getattr(mod, funcname)
-
-        if not hasattr(func, "__doc__"):
-            raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
-
-        # if not isinstance(cmd, click.BaseCommand):
-        # raise ValueError(f'lazy loading of {import_path} failed - not a click command')
-
-        return func
+    raise Exception('Background threads failed to exit after {tries}s: {threads_summary}')


-@click.group(cls=ArchiveBoxGroup, invoke_without_command=True)
-@click.option("--help", "-h", is_flag=True, help="Show help")
-@click.version_option(VERSION, "-v", "--version", package_name="archivebox", message="%(version)s")
-@click.pass_context
-def cli(ctx, help=False):
-    """ArchiveBox: The self-hosted internet archive"""
+def list_subcommands() -> Dict[str, str]:
+    """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""

-    subcommand = ArchiveBoxGroup.get_canonical_name(ctx.invoked_subcommand)
+    COMMANDS = []
+    for filename in os.listdir(CLI_DIR):
+        if is_cli_module(filename):
+            subcommand = filename.replace('archivebox_', '').replace('.py', '')
+            module = import_module('.archivebox_{}'.format(subcommand), __package__)
+            assert is_valid_cli_module(module, subcommand)
+            COMMANDS.append((subcommand, module.main.__doc__))
+            globals()[subcommand] = module.main

-    # if --help is passed or no subcommand is given, show custom help message
-    if help or ctx.invoked_subcommand is None:
-        ctx.invoke(ctx.command.get_command(ctx, "help"))
+    display_order = lambda cmd: (
+        display_first.index(cmd[0])
+        if cmd[0] in display_first else
+        100 + len(cmd[0])
+    )

-    # if the subcommand is in archive_commands or model_commands,
-    # then we need to set up the django environment and check that we're in a valid data folder
-    if subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands:
-        # print('SETUP DJANGO AND CHECK DATA FOLDER')
-        try:
-            if subcommand == "server":
-                run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes")
-                if run_in_debug:
-                    os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
-                    if "--reload" in sys.argv:
-                        os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
-                        from archivebox.config.common import get_config
-
-                        os.environ["ARCHIVEBOX_RUNSERVER_PIDFILE"] = str(get_config().TMP_DIR / "runserver.pid")
-
-            from archivebox.config.django import setup_django
-            from archivebox.misc.checks import check_data_folder
-
-            setup_django()
-            check_data_folder()
-        except Exception as e:
-            print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]", file=sys.stderr)
-            if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
-                raise
+    return dict(sorted(COMMANDS, key=display_order))


-def main(args=None, prog_name=None, stdin=None):
-    # show `docker run archivebox xyz` in help messages if running in docker
-    IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
-    IS_TTY = sys.stdin.isatty()
-    prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
+def run_subcommand(subcommand: str,
+                   subcommand_args: List[str]=None,
+                   stdin: Optional[IO]=None,
+                   pwd: Union[Path, str, None]=None) -> None:
+    """Run a given ArchiveBox subcommand with the given list of args"""

-    # stdin param allows passing input data from caller (used by __main__.py)
-    # currently not used by click-based CLI, but kept for backwards compatibility
+    subcommand_args = subcommand_args or []

-    try:
-        cli(args=args, prog_name=prog_name)
-    except KeyboardInterrupt:
-        print("\n\n[red][X] Got CTRL+C. Exiting...[/red]")
+    if subcommand not in meta_cmds:
+        from ..config import setup_django
+
+        cmd_requires_db = subcommand in archive_cmds
+        init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
+
+        if cmd_requires_db:
+            check_data_folder(pwd)
+
+        setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)
+
+        if cmd_requires_db:
+            check_migrations()
+
+    module = import_module('.archivebox_{}'.format(subcommand), __package__)
+    module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore
+
+    # wait for webhooks, signals, and other background jobs to finish before exit
+    wait_for_bg_threads_to_exit(timeout=60)
+
+
+SUBCOMMANDS = list_subcommands()
+
+class NotProvided:
+    pass
+
+
+def main(args: Optional[List[str]]=NotProvided, stdin: Optional[IO]=NotProvided, pwd: Optional[str]=None) -> None:
+    args = sys.argv[1:] if args is NotProvided else args
+    stdin = sys.stdin if stdin is NotProvided else stdin
+
+    subcommands = list_subcommands()
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description='ArchiveBox: The self-hosted internet archive',
+        add_help=False,
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--help', '-h',
+        action='store_true',
+        help=subcommands['help'],
+    )
+    group.add_argument(
+        '--version',
+        action='store_true',
+        help=subcommands['version'],
+    )
+    group.add_argument(
+        "subcommand",
+        type=str,
+        help= "The name of the subcommand to run",
+        nargs='?',
+        choices=subcommands.keys(),
+        default=None,
+    )
+    parser.add_argument(
+        "subcommand_args",
+        help="Arguments for the subcommand",
+        nargs=argparse.REMAINDER,
+    )
+    command = parser.parse_args(args or ())
+
+    if command.version:
+        command.subcommand = 'version'
+    elif command.help or command.subcommand is None:
+        command.subcommand = 'help'
+
+    if command.subcommand not in ('help', 'version', 'status'):
+        from ..logging_util import log_cli_command
+
+        log_cli_command(
+            subcommand=command.subcommand,
+            subcommand_args=command.subcommand_args,
+            stdin=stdin,
+            pwd=pwd or OUTPUT_DIR
+        )
+
+    run_subcommand(
+        subcommand=command.subcommand,
+        subcommand_args=command.subcommand_args,
+        stdin=stdin,
+        pwd=pwd or OUTPUT_DIR,
+    )
+
+
+__all__ = (
+    'SUBCOMMANDS',
+    'list_subcommands',
+    'run_subcommand',
+    *SUBCOMMANDS.keys(),
+)


-if __name__ == "__main__":
-    main()
@@ -1,307 +1,140 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
-__command__ = "archivebox add"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox add'

 import sys
-import json
-from pathlib import Path
+import argparse

-from typing import TYPE_CHECKING
+from typing import List, Optional, IO

-import rich_click as click
-
-from django.utils import timezone
-from django.db.models import QuerySet
-
-from archivebox.misc.util import enforce_types, docstring
-from archivebox.misc.util import parse_filesize_to_bytes
-from archivebox import CONSTANTS
-from archivebox.config.permissions import USER, HOSTNAME
-from archivebox.config.common import get_config
+from ..main import add
+from ..util import docstring
+from ..parsers import PARSERS
+from ..config import OUTPUT_DIR, ONLY_NEW
+from ..logging_util import SmartFormatter, accept_stdin, stderr


-if TYPE_CHECKING:
-    from archivebox.crawls.models import Crawl
-    from archivebox.core.models import Snapshot
-
-
-def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
-    from archivebox.misc.jsonl import read_args_or_stdin
-
-    urls: list[str] = []
-    for record in read_args_or_stdin(args):
-        url = record.get("url")
-        if isinstance(url, str) and url:
-            urls.append(url)
-
-        urls_field = record.get("urls")
-        if isinstance(urls_field, str):
-            for line in urls_field.splitlines():
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    urls.append(line)
-
-    return urls
-
-
-@enforce_types
-def add(
-    urls: str | list[str],
-    snapshot_ids: list[str] | None = None,
-    depth: int | str = 0,
-    max_urls: int = 0,
-    crawl_max_size: int | str = 0,
-    snapshot_max_size: int | str = 0,
-    crawl_max_concurrent_snapshots: int | None = None,
-    tag: str = "",
-    url_allowlist: str = "",
-    url_denylist: str = "",
-    parser: str = "auto",
-    plugins: str = "",
-    persona: str = "Default",
-    overwrite: bool = False,
-    update: bool | None = None,
-    index_only: bool = False,
-    bg: bool = False,
-    created_by_id: int | None = None,
-) -> tuple["Crawl", QuerySet["Snapshot"]]:
-    """Add a new URL or list of URLs to your archive.
-
-    The flow is:
-    1. Save URLs to sources file
-    2. Create Crawl with URLs and max_depth
-    3. Crawl runner creates Snapshots from Crawl URLs (depth=0)
-    4. Crawl runner runs parser extractors on root snapshots
-    5. Parser extractors output to urls.jsonl
-    6. URLs are added to Crawl.urls and child Snapshots are created
-    7. Repeat until max_depth is reached
-    """
-
-    from rich import print
-
-    depth = int(depth)
-    max_urls = int(max_urls or 0)
-    crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
-    snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    config = get_config()
-    if crawl_max_concurrent_snapshots is None:
-        crawl_max_concurrent_snapshots = config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
-    crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
-
-    if depth not in (0, 1, 2, 3, 4):
-        raise ValueError("Depth must be 0-4")
-    if max_urls < 0:
-        raise ValueError("max_urls must be >= 0")
-    if crawl_max_size < 0:
-        raise ValueError("crawl_max_size must be >= 0")
-    if snapshot_max_size < 0:
-        raise ValueError("snapshot_max_size must be >= 0")
-    if crawl_max_concurrent_snapshots < 1:
-        raise ValueError("crawl_max_concurrent_snapshots must be >= 1")
-
-    # import models once django is set up
-    from archivebox.crawls.models import Crawl
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.personas.models import Persona
-    from archivebox.misc.logging_util import printable_filesize
-    from archivebox.misc.system import get_dir_size
-    from archivebox.services.runner import run_crawl
-
-    created_by_id = created_by_id or get_or_create_system_user_pk()
-    started_at = timezone.now()
-    if update is None:
-        update = not config.ONLY_NEW
-
-    if isinstance(urls, str):
-        url_list = [line.strip() for line in urls.splitlines() if line.strip()]
-    else:
-        url_list = [str(url).strip() for url in urls if str(url).strip()]
-    if snapshot_ids and len(snapshot_ids) != len(url_list):
-        raise ValueError("snapshot_ids length must match urls length")
-
-    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
-    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
-    sources_file.parent.mkdir(parents=True, exist_ok=True)
-    if snapshot_ids:
-        sources_file.write_text(
-            "\n".join(json.dumps({"url": url, "id": snapshot_ids[index], "tags": tag, "depth": 0}) for index, url in enumerate(url_list)),
+@docstring(add.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=add.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--tag', '-t',
+        type=str,
+        default='',
+        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
+    )
+    parser.add_argument(
+        '--update', #'-u',
+        action='store_true',
+        default=not ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
+        help="Also retry previously skipped/failed links when adding new links",
+    )
+    parser.add_argument(
+        '--update-all', #'-n',
+        action='store_true',
+        default=False, 
+        help="Also update ALL links in index when finished adding new links",
+    )
+    parser.add_argument(
+        '--index-only', #'-o',
+        action='store_true',
+        help="Add the links to the main index without archiving them",
+    )
+    parser.add_argument(
+        'urls',
+        nargs='*',
+        type=str,
+        default=None,
+        help=(
+            'URLs or paths to archive e.g.:\n'
+            '    https://getpocket.com/users/USERNAME/feed/all\n'
+            '    https://example.com/some/rss/feed.xml\n'
+            '    https://example.com\n'
+            '    ~/Downloads/firefox_bookmarks_export.html\n'
+            '    ~/Desktop/sites_list.csv\n'
        )
-    else:
-        sources_file.write_text("\n".join(url_list))
+    )
+    parser.add_argument(
+        "--depth",
+        action="store",
+        default=0,
+        choices=[0, 1],
+        type=int,
+        help="Recursively archive all linked pages up to this many hops away"
+    )
+    parser.add_argument(
+        "--overwrite",
+        default=False,
+        action="store_true",
+        help="Re-archive URLs from scratch, overwriting any existing files"
+    )
+    parser.add_argument(
+        "--init", #'-i',
+        action='store_true',
+        help="Init/upgrade the curent data directory before adding",
+    )
+    parser.add_argument(
+        "--extract",
+        type=str,
+        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
+              This does not take precedence over the configuration",
+        default=""
+    )
+    parser.add_argument(
+        "--parser",
+        type=str,
+        help="Parser used to read inputted URLs.",
+        default="auto",
+        choices=["auto", *PARSERS.keys()],
+    )
+    command = parser.parse_args(args or ())
+    urls = command.urls

-    # 2. Create a new Crawl with inline URLs
-    cli_args = [*sys.argv]
-    if cli_args[0].lower().endswith("archivebox"):
-        cli_args[0] = "archivebox"
-    cmd_str = " ".join(cli_args)
+    stdin_urls = ''
+    if not urls:
+        stdin_urls = accept_stdin(stdin)

-    timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
-
-    # Read URLs directly into crawl
-    urls_content = sources_file.read_text()
-    persona_name = (persona or "Default").strip() or "Default"
-    plugins = plugins or str(config.get("PLUGINS") or "")
-    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
-    persona_obj.ensure_dirs()
-
-    crawl = Crawl.objects.create(
-        urls=urls_content,
-        max_depth=depth,
-        max_urls=max_urls,
-        crawl_max_size=crawl_max_size,
-        snapshot_max_size=snapshot_max_size,
-        tags_str=tag,
-        persona_id=persona_obj.id,
-        label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
-        created_by_id=created_by_id,
-        status=Crawl.StatusChoices.QUEUED if bg or index_only else Crawl.StatusChoices.STARTED,
-        retry_at=timezone.now() if bg else None,
-        config={
-            "ONLY_NEW": not update,
-            "INDEX_ONLY": index_only,
-            "OVERWRITE": overwrite,
-            "PLUGINS": plugins,
-            "DEFAULT_PERSONA": persona_name,
-            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots,
-            "PARSER": parser,
-            **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
-            **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
-        },
+    if (stdin_urls and urls) or (not stdin and not urls):
+        stderr(
+            '[X] You must pass URLs/paths to add via stdin or CLI arguments.\n',
+            color='red',
+        )
+        raise SystemExit(2)
+    add(
+        urls=stdin_urls or urls,
+        depth=command.depth,
+        tag=command.tag,
+        update=command.update,
+        update_all=command.update_all,
+        index_only=command.index_only,
+        overwrite=command.overwrite,
+        init=command.init,
+        extractors=command.extract,
+        parser=command.parser,
+        out_dir=pwd or OUTPUT_DIR,
    )

-    print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
-    first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
-    print(f"    [dim]First URL: {first_url}[/dim]")

-    # 3. The CrawlMachine will create Snapshots from all URLs when started
-    #    Parser extractors run on snapshots and discover more URLs
-    #    Discovered URLs become child Snapshots (depth+1)
-
-    if index_only:
-        # Just create the crawl but don't start processing
-        print("[yellow]\\[*] Index-only mode - crawl created but not started[/yellow]")
-        crawl.create_snapshots_from_urls()
-        return crawl, crawl.snapshot_set.all()
-
-    if bg:
-        crawl.create_snapshots_from_urls()
-
-    # 5. Start the crawl runner to process the queue
-    #    The runner will:
-    #    - Process Crawl -> create Snapshots from all URLs
-    #    - Process Snapshots -> run extractors
-    #    - Parser extractors discover new URLs -> create child Snapshots
-    #    - Repeat until max_depth reached
-
-    if bg:
-        # Background mode: just queue work and return (background runner via server will pick it up)
-        print(
-            "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
-        )
-    else:
-        # Foreground mode: run full crawl runner until all work is done
-        print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
-        run_crawl(str(crawl.id))
-
-        # Print summary for foreground runs
-        try:
-            crawl.refresh_from_db()
-            snapshots_count = crawl.snapshot_set.count()
-            try:
-                from django.db.models import Count, Sum
-
-                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("archiveresult__output_size"))
-                total_bytes = int(totals["total_bytes"] or 0) if totals["snapshot_count"] else 0
-            except Exception:
-                total_bytes, _, _ = get_dir_size(crawl.output_dir)
-            total_size = printable_filesize(total_bytes)
-            total_time = timezone.now() - started_at
-            total_seconds = int(total_time.total_seconds())
-            mins, secs = divmod(total_seconds, 60)
-            hours, mins = divmod(mins, 60)
-            if hours:
-                duration_str = f"{hours}h {mins}m {secs}s"
-            elif mins:
-                duration_str = f"{mins}m {secs}s"
-            else:
-                duration_str = f"{secs}s"
-
-            # Output dir relative to DATA_DIR
-            try:
-                rel_output = Path(crawl.output_dir).relative_to(CONSTANTS.DATA_DIR)
-                rel_output_str = f"./{rel_output}"
-            except Exception:
-                rel_output_str = str(crawl.output_dir)
-
-            bind_addr = config.BIND_ADDR or "127.0.0.1:8000"
-            if bind_addr.startswith("http://") or bind_addr.startswith("https://"):
-                base_url = bind_addr
-            else:
-                base_url = f"http://{bind_addr}"
-            admin_url = f"{base_url}/admin/crawls/crawl/{crawl.id}/change/"
-
-            print("\n[bold]crawl output saved to:[/bold]")
-            print(f"  {rel_output_str}")
-            print(f"  {admin_url}")
-            print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
-            print(f"[bold]total size:[/bold] {total_size}")
-            print(f"[bold]total time:[/bold] {duration_str}")
-        except Exception:
-            # Summary is best-effort; avoid failing the command if something goes wrong
-            pass
-
-    # 6. Return the list of Snapshots in this crawl
-    snapshots = crawl.snapshot_set.all()
-    return crawl, snapshots
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)


-@click.command()
-@click.option(
-    "--depth",
-    "-d",
-    type=click.Choice([str(i) for i in range(5)]),
-    default="0",
-    help="Recursively archive linked pages up to N hops away",
-)
-@click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
-@click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
-@click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)")
-@click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl")
-@click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
-@click.option("--url-allowlist", "--domain-allowlist", default="", help="Comma-separated URL/domain allowlist for this crawl")
-@click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
-@click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
-@click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
-@click.option("--persona", default="Default", help="Authentication profile to use when archiving")
-@click.option("--overwrite", "-F", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
-@click.option("--update", is_flag=True, default=None, help="Retry any previously skipped/failed URLs when re-adding them")
-@click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
-@click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
-@click.argument("urls", nargs=-1, type=click.Path())
-@docstring(add.__doc__)
-def main(**kwargs):
-    """Add a new URL or list of URLs to your archive"""
-
-    raw_urls = kwargs.pop("urls")
-    urls = _collect_input_urls(raw_urls)
-    if not urls:
-        raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
-    if int(kwargs.get("max_urls") or 0) < 0:
-        raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
-    try:
-        kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
-    except ValueError as err:
-        raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
-    try:
-        kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
-    except ValueError as err:
-        raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
-    if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
-        raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
-
-    add(urls=urls, **kwargs)
-
-
-if __name__ == "__main__":
-    main()
+# TODO: Implement these
+#
+# parser.add_argument(
+#     '--mirror', #'-m',
+#     action='store_true',
+#     help='Archive an entire site (finding all linked pages below it on the same domain)',
+# )
+# parser.add_argument(
+#     '--crawler', #'-r',
+#     choices=('depth_first', 'breadth_first'),
+#     help='Controls which crawler to use in order to find outlinks in a given page',
+#     default=None,
+# )
@@ -1,387 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox archiveresult <action> [args...] [--filters]
-
-Manage ArchiveResult records (plugin extraction results).
-
-Actions:
-    create  - Create ArchiveResults for Snapshots (queue extractions)
-    list    - List ArchiveResults as JSONL (with optional filters)
-    update  - Update ArchiveResults from stdin JSONL
-    delete  - Delete ArchiveResults from stdin JSONL
-
-Examples:
-    # Create ArchiveResults for snapshots (queue for extraction)
-    archivebox snapshot list --status=queued | archivebox archiveresult create
-    archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
-
-    # List with filters
-    archivebox archiveresult list --status=failed
-    archivebox archiveresult list --plugin=screenshot --status=succeeded
-
-    # Update (reset failed extractions to queued)
-    archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued
-
-    # Delete
-    archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
-
-    # Re-run failed extractions
-    archivebox archiveresult list --status=failed | archivebox run
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox archiveresult"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_utils import apply_filters
-
-
-def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
-    return {
-        "type": "ArchiveResult",
-        "snapshot_id": str(snapshot_id),
-        "plugin": plugin,
-        "hook_name": hook_name,
-        "status": status,
-    }
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_archiveresults(
-    snapshot_id: str | None = None,
-    plugin: str | None = None,
-    status: str = "queued",
-) -> int:
-    """
-    Create ArchiveResult request records for Snapshots.
-
-    Reads Snapshot records from stdin and emits ArchiveResult request JSONL.
-    Pass-through: Non-Snapshot/ArchiveResult records are output unchanged.
-    If --plugin is specified, only emits requests for that plugin.
-    Otherwise, emits requests for all enabled snapshot hooks.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.config.common import get_config
-    from archivebox.hooks import discover_hooks
-    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
-    from archivebox.core.models import Snapshot
-
-    is_tty = sys.stdout.isatty()
-
-    # If snapshot_id provided directly, use that
-    if snapshot_id:
-        try:
-            snapshots = [Snapshot.objects.get(id=snapshot_id)]
-            pass_through_records = []
-        except Snapshot.DoesNotExist:
-            rprint(f"[red]Snapshot not found: {snapshot_id}[/red]", file=sys.stderr)
-            return 1
-    else:
-        # Read from stdin
-        records = list(read_stdin())
-        if not records:
-            rprint("[yellow]No Snapshot records provided via stdin[/yellow]", file=sys.stderr)
-            return 1
-
-        # Separate snapshot records from pass-through records
-        snapshot_ids = []
-        pass_through_records = []
-
-        for record in records:
-            record_type = record.get("type", "")
-
-            if record_type == TYPE_SNAPSHOT:
-                # Pass through the Snapshot record itself
-                pass_through_records.append(record)
-                if record.get("id"):
-                    snapshot_ids.append(record["id"])
-
-            elif record_type == TYPE_ARCHIVERESULT:
-                # ArchiveResult records: pass through if they have an id
-                if record.get("id"):
-                    pass_through_records.append(record)
-                # If no id, we could create it, but for now just pass through
-                else:
-                    pass_through_records.append(record)
-
-            elif record_type:
-                # Other typed records (Crawl, Tag, etc): pass through
-                pass_through_records.append(record)
-
-            elif record.get("id"):
-                # Untyped record with id - assume it's a snapshot ID
-                snapshot_ids.append(record["id"])
-
-        # Output pass-through records first
-        if not is_tty:
-            for record in pass_through_records:
-                write_record(record)
-
-        if not snapshot_ids:
-            if pass_through_records:
-                rprint(f"[dim]Passed through {len(pass_through_records)} records, no new snapshots to process[/dim]", file=sys.stderr)
-                return 0
-            rprint("[yellow]No valid Snapshot IDs in input[/yellow]", file=sys.stderr)
-            return 1
-
-        snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids))
-
-    if not snapshots:
-        rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
-        return 0 if pass_through_records else 1
-
-    created_count = 0
-    for snapshot in snapshots:
-        if plugin:
-            if not is_tty:
-                write_record(build_archiveresult_request(snapshot.id, plugin, status=status))
-            created_count += 1
-        else:
-            config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
-            hooks = discover_hooks("Snapshot", config=config)
-            for hook_path in hooks:
-                hook_name = hook_path.name
-                plugin_name = hook_path.parent.name
-                if not is_tty:
-                    write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
-                created_count += 1
-
-    rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_archiveresults(
-    status: str | None = None,
-    plugin: str | None = None,
-    snapshot_id: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List ArchiveResults as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import ArchiveResult
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = ArchiveResult.objects.all().order_by("-start_ts")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "plugin": plugin,
-        "snapshot_id": snapshot_id,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for result in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "succeeded": "green",
-                "failed": "red",
-                "skipped": "dim",
-                "noresults": "dim",
-                "backoff": "magenta",
-            }.get(result.status, "dim")
-            rprint(
-                f"[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
-            )
-        else:
-            write_record(result.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} archive results[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_archiveresults(
-    status: str | None = None,
-) -> int:
-    """
-    Update ArchiveResults from stdin JSONL.
-
-    Reads ArchiveResult records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.core.models import ArchiveResult
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        result_id = record.get("id")
-        if not result_id:
-            continue
-
-        try:
-            result = ArchiveResult.objects.get(id=result_id)
-
-            # Apply updates from CLI flags
-            if status:
-                result.status = status
-
-            result.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(result.to_json())
-
-        except ArchiveResult.DoesNotExist:
-            rprint(f"[yellow]ArchiveResult not found: {result_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} archive results[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete ArchiveResults from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.core.models import ArchiveResult
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    result_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not result_ids:
-        rprint("[yellow]No valid archive result IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    results = ArchiveResult.objects.filter(id__in=result_ids)
-    count = results.count()
-
-    if count == 0:
-        rprint("[yellow]No matching archive results found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} archive results (dry run)[/yellow]", file=sys.stderr)
-        for result in results[:10]:
-            rprint(f"  [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}", file=sys.stderr)
-        if count > 10:
-            rprint(f"  ... and {count - 10} more", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = results.delete()
-    rprint(f"[green]Deleted {deleted_count} archive results[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage ArchiveResult records (plugin extraction results)."""
-    pass
-
-
-@main.command("create")
-@click.option("--snapshot-id", help="Snapshot ID to create results for")
-@click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
-    """Create ArchiveResults for Snapshots from stdin JSONL."""
-    sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
-
-
-@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, succeeded, failed, skipped)")
-@click.option("--plugin", "-p", help="Filter by plugin name")
-@click.option("--snapshot-id", help="Filter by snapshot ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(
-    status: str | None,
-    plugin: str | None,
-    snapshot_id: str | None,
-    limit: int | None,
-):
-    """List ArchiveResults as JSONL."""
-    sys.exit(
-        list_archiveresults(
-            status=status,
-            plugin=plugin,
-            snapshot_id=snapshot_id,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--status", "-s", help="Set status")
-def update_cmd(status: str | None):
-    """Update ArchiveResults from stdin JSONL."""
-    sys.exit(update_archiveresults(status=status))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete ArchiveResults from stdin JSONL."""
-    sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,319 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox binary <action> [args...] [--filters]
-
-Manage Binary records (detected executables like chrome, wget, etc.).
-
-Actions:
-    create  - Create/register a Binary
-    list    - List Binaries as JSONL (with optional filters)
-    update  - Update Binaries from stdin JSONL
-    delete  - Delete Binaries from stdin JSONL
-
-Examples:
-    # List all binaries
-    archivebox binary list
-
-    # List specific binary
-    archivebox binary list --name=chrome
-
-    # List binaries with specific version
-    archivebox binary list --version__icontains=120
-
-    # Delete old binary entries
-    archivebox binary list --name=chrome | archivebox binary delete --yes
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox binary"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_utils import apply_filters
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_binary(
-    name: str,
-    abspath: str,
-    version: str = "",
-) -> int:
-    """
-    Create/register a Binary.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    if not name or not abspath:
-        rprint("[red]Both --name and --abspath are required[/red]", file=sys.stderr)
-        return 1
-
-    try:
-        from archivebox.machine.models import Machine
-
-        machine = Machine.current()
-        created = not Binary.objects.filter(
-            machine=machine,
-            name=name,
-            abspath=abspath,
-            version=version,
-        ).exists()
-
-        # Mirror the Binary model lifecycle used elsewhere in the system so CLI
-        # records are owned by the current machine and can be safely piped into
-        # `archivebox run` without creating invalid rows missing machine_id.
-        binary = Binary.from_json(
-            {
-                "name": name,
-                "abspath": abspath,
-                "version": version,
-                "binproviders": "env",
-                "binprovider": "env",
-            },
-        )
-        if binary is None:
-            raise ValueError("failed to create binary record")
-
-        if not is_tty:
-            write_record(binary.to_json())
-
-        if created:
-            rprint(f"[green]Created binary: {name} at {abspath}[/green]", file=sys.stderr)
-        else:
-            rprint(f"[dim]Binary already exists: {name} at {abspath}[/dim]", file=sys.stderr)
-
-        return 0
-
-    except Exception as e:
-        rprint(f"[red]Error creating binary: {e}[/red]", file=sys.stderr)
-        return 1
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_binaries(
-    name: str | None = None,
-    abspath__icontains: str | None = None,
-    version__icontains: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Binaries as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Binary.objects.all().order_by("name", "-modified_at", "-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "name": name,
-        "abspath__icontains": abspath__icontains,
-        "version__icontains": version__icontains,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for binary in queryset:
-        if is_tty:
-            rprint(f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}")
-        else:
-            write_record(binary.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} binaries[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_binaries(
-    version: str | None = None,
-    abspath: str | None = None,
-) -> int:
-    """
-    Update Binaries from stdin JSONL.
-
-    Reads Binary records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        binary_id = record.get("id")
-        if not binary_id:
-            continue
-
-        try:
-            binary = Binary.objects.get(id=binary_id)
-
-            # Apply updates from CLI flags
-            if version:
-                binary.version = version
-            if abspath:
-                binary.abspath = abspath
-
-            binary.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(binary.to_json())
-
-        except Binary.DoesNotExist:
-            rprint(f"[yellow]Binary not found: {binary_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} binaries[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_binaries(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Binaries from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.machine.models import Binary
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    binary_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not binary_ids:
-        rprint("[yellow]No valid binary IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    binaries = Binary.objects.filter(id__in=binary_ids)
-    count = binaries.count()
-
-    if count == 0:
-        rprint("[yellow]No matching binaries found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} binaries (dry run)[/yellow]", file=sys.stderr)
-        for binary in binaries:
-            rprint(f"  {binary.name} {binary.abspath}", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = binaries.delete()
-    rprint(f"[green]Deleted {deleted_count} binaries[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Binary records (detected executables)."""
-    pass
-
-
-@main.command("create")
-@click.option("--name", "-n", required=True, help="Binary name (e.g., chrome, wget)")
-@click.option("--abspath", "-p", required=True, help="Absolute path to binary")
-@click.option("--version", "-v", default="", help="Binary version")
-def create_cmd(name: str, abspath: str, version: str):
-    """Create/register a Binary."""
-    sys.exit(create_binary(name=name, abspath=abspath, version=version))
-
-
-@main.command("list")
-@click.option("--name", "-n", help="Filter by name")
-@click.option("--abspath__icontains", help="Filter by path contains")
-@click.option("--version__icontains", help="Filter by version contains")
-@click.option("--limit", type=int, help="Limit number of results")
-def list_cmd(
-    name: str | None,
-    abspath__icontains: str | None,
-    version__icontains: str | None,
-    limit: int | None,
-):
-    """List Binaries as JSONL."""
-    sys.exit(
-        list_binaries(
-            name=name,
-            abspath__icontains=abspath__icontains,
-            version__icontains=version__icontains,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--version", "-v", help="Set version")
-@click.option("--abspath", "-p", help="Set path")
-def update_cmd(version: str | None, abspath: str | None):
-    """Update Binaries from stdin JSONL."""
-    sys.exit(update_binaries(version=version, abspath=abspath))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Binaries from stdin JSONL."""
-    sys.exit(delete_binaries(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,190 +1,64 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox config'

 import sys
-import toml
-import rich_click as click
-from rich import print
+import argparse

-from archivebox.misc.util import docstring, enforce_types
-from archivebox.misc.toml_util import CustomTOMLEncoder
+from typing import Optional, List, IO
+
+from ..main import config
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, accept_stdin


-def _format_toml(config: dict) -> str:
-    return toml.dumps(config, encoder=CustomTOMLEncoder()).strip().replace("\n\n", "\n")
-
-
-@enforce_types
-def config(
-    *keys,
-    get: bool = False,
-    set: bool = False,
-    search: bool = False,
-    reset: bool = False,
-    **kwargs,
-) -> None:
-    """Get and set your ArchiveBox project configuration values"""
-
-    from archivebox.misc.checks import check_data_folder
-    from archivebox.misc.logging_util import printable_config
-    from abx_plugins.plugins.base.utils import resolve_alias
-    from archivebox.config.collection import write_config_file
-    from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
-    from archivebox.hooks import discover_plugin_configs
-
-    check_data_folder()
-
-    FLAT_CONFIG = get_config().as_dict()
-    CONFIGS = get_all_configs()
-    plugin_schemas = {
-        plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
-    }
-    core_config_aliases = {
-        alias.upper(): field_name
-        for field_name, field in ArchiveBoxConfig.model_fields.items()
-        for alias in (field_name, str(field.alias or ""))
-        if alias
-    }
-
-    config_options: list[str] = list(kwargs.pop("key=value", []) or keys or [f"{key}={val}" for key, val in kwargs.items()])
-    no_args = not (get or set or reset or config_options)
-
-    matching_config = {}
-    if search:
-        if config_options:
-            config_options = [
-                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
-            ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            for config_section in CONFIGS.values():
-                aliases = getattr(config_section, "aliases", {})
-
-                for search_key in config_options:
-                    # search all aliases in the section
-                    for alias_key, key in aliases.items():
-                        if search_key.lower() in alias_key.lower():
-                            matching_config[key] = dict(config_section)[key]
-
-                    # search all keys and values in the section
-                    for existing_key, value in dict(config_section).items():
-                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
-                            matching_config[existing_key] = value
-
-        print(printable_config(matching_config))
-        raise SystemExit(not matching_config)
-
-    elif get or no_args:
-        if config_options:
-            config_options = [
-                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
-            ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
-            if failed_config:
-                print("\n[red][X] These options failed to get[/red]")
-                print("    {}".format("\n    ".join(config_options)))
-                raise SystemExit(1)
-        else:
-            matching_config = FLAT_CONFIG
-
-        # Display core config sections
-        for config_section in CONFIGS.values():
-            section_header = getattr(config_section, "toml_section_header", "")
-            if isinstance(section_header, str) and section_header:
-                print(f"[grey53]\\[{section_header}][/grey53]")
-            else:
-                print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
-
-            kv_in_section = {key: val for key, val in dict(config_section).items() if key in matching_config}
-            print(_format_toml(kv_in_section))
-            print("[grey53]################################################################[/grey53]")
-
-        plugin_keys = {}
-
-        # Collect all plugin config keys
-        for schema in plugin_schemas.values():
-            for key in schema.keys():
-                if key in matching_config:
-                    plugin_keys[key] = matching_config[key]
-
-        # Display all plugin config in single [PLUGINS] section
-        if plugin_keys:
-            print("[grey53]\\[PLUGINS][/grey53]")
-            print(_format_toml(plugin_keys))
-            print("[grey53]################################################################[/grey53]")
-
-        raise SystemExit(not matching_config)
-
-    elif set:
-        new_config = {}
-        failed_options = []
-        for line in config_options:
-            if line.startswith("#") or not line.strip():
-                continue
-            if "=" not in line:
-                print("[red][X] Config KEY=VALUE must have an = sign in it[/red]")
-                print(f"    {line}")
-                raise SystemExit(2)
-
-            raw_key, val = line.split("=", 1)
-            raw_key = raw_key.upper().strip()
-            key = core_config_aliases.get(raw_key) or resolve_alias(raw_key, plugin_schemas)
-            if key != raw_key:
-                print(
-                    f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
-                )
-
-            if key in FLAT_CONFIG:
-                new_config[key] = val.strip()
-            else:
-                failed_options.append(line)
-
-        if new_config:
-            before = FLAT_CONFIG
-            matching_config = write_config_file(new_config)
-            after = get_config().as_dict()
-            print(printable_config(matching_config))
-
-            side_effect_changes = {}
-            for key, val in after.items():
-                if key in FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
-                    side_effect_changes[key] = after[key]
-
-            if side_effect_changes:
-                print(file=sys.stderr)
-                print("[yellow][i] Note: This change also affected these other options that depended on it:[/yellow]", file=sys.stderr)
-                print("    {}".format(printable_config(side_effect_changes, prefix="    ")), file=sys.stderr)
-
-        if failed_options:
-            print()
-            print("[red][X] These options failed to set (check for typos):[/red]")
-            print("    {}".format("\n    ".join(failed_options)))
-            raise SystemExit(1)
-
-    elif reset:
-        print("[red][X] This command is not implemented yet.[/red]")
-        print("    Please manually remove the relevant lines from your config file:")
-        raise SystemExit(2)
-
-    else:
-        print("[red][X] You must pass either --get or --set, or no arguments to get the whole config.[/red]")
-        print("    archivebox config")
-        print("    archivebox config --get SOME_KEY")
-        print("    archivebox config --set SOME_KEY=SOME_VALUE")
-        raise SystemExit(2)
-
-
-@click.command()
-@click.option("--search", is_flag=True, help="Search config KEYs, VALUEs, and ALIASES for the given term")
-@click.option("--get", is_flag=True, help="Get the value for the given config KEYs")
-@click.option("--set", is_flag=True, help="Set the given KEY=VALUE config values")
-@click.option("--reset", is_flag=True, help="Reset the given KEY config values to their defaults")
-@click.argument("KEY=VALUE", nargs=-1, type=str)
@docstring(config.__doc__)
-def main(**kwargs) -> None:
-    config(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=config.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--get', #'-g',
+        action='store_true',
+        help="Get the value for the given config KEYs",
+    )
+    group.add_argument(
+        '--set', #'-s',
+        action='store_true',
+        help="Set the given KEY=VALUE config values",
+    )
+    group.add_argument(
+        '--reset', #'-s',
+        action='store_true',
+        help="Reset the given KEY config values to their defaults",
+    )
+    parser.add_argument(
+        'config_options',
+        nargs='*',
+        type=str,
+        help='KEY or KEY=VALUE formatted config values to get or set',
+    )
+    command = parser.parse_args(args or ())
+
+    config_options_str = ''
+    if not command.config_options:
+        config_options_str = accept_stdin(stdin)
+
+    config(
+        config_options_str=config_options_str,
+        config_options=command.config_options,
+        get=command.get,
+        set=command.set,
+        reset=command.reset,
+        out_dir=pwd or OUTPUT_DIR,
+    )


-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,384 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox crawl <action> [args...] [--filters]
-
-Manage Crawl records.
-
-Actions:
-    create  - Create Crawl jobs from URLs
-    list    - List Crawls as JSONL (with optional filters)
-    update  - Update Crawls from stdin JSONL
-    delete  - Delete Crawls from stdin JSONL
-
-Examples:
-    # Create
-    archivebox crawl create https://example.com https://foo.com --depth=1
-    archivebox crawl create --tag=news https://example.com
-
-    # List with filters
-    archivebox crawl list --status=queued
-    archivebox crawl list --urls__icontains=example.com
-
-    # Update
-    archivebox crawl list --status=started | archivebox crawl update --status=queued
-
-    # Delete
-    archivebox crawl list --urls__icontains=spam.com | archivebox crawl delete --yes
-
-    # Full pipeline
-    archivebox crawl create https://example.com | archivebox snapshot create | archivebox run
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-from collections.abc import Iterable
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_utils import apply_filters
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_crawl(
-    urls: Iterable[str],
-    depth: int = 0,
-    tag: str = "",
-    status: str = "queued",
-    created_by_id: int | None = None,
-) -> int:
-    """
-    Create a Crawl job from URLs.
-
-    Takes URLs as args or stdin, creates one Crawl with all URLs, outputs JSONL.
-    Pass-through: Records that are not URLs are output unchanged (for piping).
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.crawls.models import Crawl
-
-    created_by_id = created_by_id or get_or_create_system_user_pk()
-    is_tty = sys.stdout.isatty()
-
-    # Collect all input records
-    records = list(read_args_or_stdin(urls))
-
-    if not records:
-        rprint("[yellow]No URLs provided. Pass URLs as arguments or via stdin.[/yellow]", file=sys.stderr)
-        return 1
-
-    # Separate pass-through records from URL records
-    url_list = []
-    pass_through_records = []
-
-    for record in records:
-        record_type = record.get("type", "")
-
-        # Pass-through: output records that aren't URL/Crawl types
-        if record_type and record_type != TYPE_CRAWL and not record.get("url") and not record.get("urls"):
-            pass_through_records.append(record)
-            continue
-
-        # Handle existing Crawl records (just pass through with id)
-        if record_type == TYPE_CRAWL and record.get("id"):
-            pass_through_records.append(record)
-            continue
-
-        # Collect URLs
-        url = record.get("url")
-        if url:
-            url_list.append(url)
-
-        # Handle 'urls' field (newline-separated)
-        urls_field = record.get("urls")
-        if urls_field:
-            for line in urls_field.split("\n"):
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    url_list.append(line)
-
-    # Output pass-through records first
-    if not is_tty:
-        for record in pass_through_records:
-            write_record(record)
-
-    if not url_list:
-        if pass_through_records:
-            # If we had pass-through records but no URLs, that's OK
-            rprint(f"[dim]Passed through {len(pass_through_records)} records, no new URLs[/dim]", file=sys.stderr)
-            return 0
-        rprint("[red]No valid URLs found[/red]", file=sys.stderr)
-        return 1
-
-    try:
-        # Build crawl record with all URLs as newline-separated string
-        crawl_record = {
-            "urls": "\n".join(url_list),
-            "max_depth": depth,
-            "tags_str": tag,
-            "status": status,
-            "label": "",
-        }
-
-        crawl = Crawl.from_json(crawl_record, overrides={"created_by_id": created_by_id})
-        if not crawl:
-            rprint("[red]Failed to create crawl[/red]", file=sys.stderr)
-            return 1
-
-        # Output JSONL record (only when piped)
-        if not is_tty:
-            write_record(crawl.to_json())
-
-        rprint(f"[green]Created crawl with {len(url_list)} URLs[/green]", file=sys.stderr)
-
-        # If TTY, show human-readable output
-        if is_tty:
-            rprint(f"  [dim]{crawl.id}[/dim]", file=sys.stderr)
-            for url in url_list[:5]:  # Show first 5 URLs
-                rprint(f"    {url[:70]}", file=sys.stderr)
-            if len(url_list) > 5:
-                rprint(f"    ... and {len(url_list) - 5} more", file=sys.stderr)
-
-        return 0
-
-    except Exception as e:
-        rprint(f"[red]Error creating crawl: {e}[/red]", file=sys.stderr)
-        return 1
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_crawls(
-    status: str | None = None,
-    urls__icontains: str | None = None,
-    max_depth: int | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Crawls as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.crawls.models import Crawl
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Crawl.objects.all().order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "urls__icontains": urls__icontains,
-        "max_depth": max_depth,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for crawl in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(crawl.status, "dim")
-            url_preview = crawl.urls[:50].replace("\n", " ")
-            rprint(f"[{status_color}]{crawl.status:8}[/{status_color}] [dim]{crawl.id}[/dim] {url_preview}...")
-        else:
-            write_record(crawl.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} crawls[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_crawls(
-    status: str | None = None,
-    max_depth: int | None = None,
-) -> int:
-    """
-    Update Crawls from stdin JSONL.
-
-    Reads Crawl records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from django.utils import timezone
-
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.crawls.models import Crawl
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        crawl_id = record.get("id")
-        if not crawl_id:
-            continue
-
-        try:
-            crawl = Crawl.objects.get(id=crawl_id)
-
-            # Apply updates from CLI flags
-            if status:
-                crawl.status = status
-                crawl.retry_at = timezone.now()
-            if max_depth is not None:
-                crawl.max_depth = max_depth
-
-            crawl.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(crawl.to_json())
-
-        except Crawl.DoesNotExist:
-            rprint(f"[yellow]Crawl not found: {crawl_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} crawls[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_crawls(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Crawls from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.crawls.models import Crawl
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    crawl_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not crawl_ids:
-        rprint("[yellow]No valid crawl IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    crawls = Crawl.objects.filter(id__in=crawl_ids)
-    count = crawls.count()
-
-    if count == 0:
-        rprint("[yellow]No matching crawls found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} crawls (dry run)[/yellow]", file=sys.stderr)
-        for crawl in crawls:
-            url_preview = crawl.urls[:50].replace("\n", " ")
-            rprint(f"  [dim]{crawl.id}[/dim] {url_preview}...", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = crawls.delete()
-    rprint(f"[green]Deleted {deleted_count} crawls[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Crawl records."""
-    pass
-
-
-@main.command("create")
-@click.argument("urls", nargs=-1)
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(urls: tuple, depth: int, tag: str, status: str):
-    """Create a Crawl job from URLs or stdin."""
-    sys.exit(create_crawl(urls, depth=depth, tag=tag, status=status))
-
-
-@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--urls__icontains", help="Filter by URLs contains")
-@click.option("--max-depth", type=int, help="Filter by max depth")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(
-    status: str | None,
-    urls__icontains: str | None,
-    max_depth: int | None,
-    limit: int | None,
-):
-    """List Crawls as JSONL."""
-    sys.exit(
-        list_crawls(
-            status=status,
-            urls__icontains=urls__icontains,
-            max_depth=max_depth,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--status", "-s", help="Set status")
-@click.option("--max-depth", type=int, help="Set max depth")
-def update_cmd(status: str | None, max_depth: int | None):
-    """Update Crawls from stdin JSONL."""
-    sys.exit(update_crawls(status=status, max_depth=max_depth))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Crawls from stdin JSONL."""
-    sys.exit(delete_crawls(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_add import add
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
-@click.argument("urls", nargs=-1)
-def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
-    del status, wait
-    add(list(urls), depth=depth, tag=tag, index_only=True, bg=True)
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()
@@ -1,311 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox extract [snapshot_ids...] [--plugins=NAMES]
-
-Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL.
-
-Input formats:
-    - Snapshot UUIDs (one per line)
-    - JSONL: {"type": "Snapshot", "id": "...", "url": "..."}
-    - JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."}
-
-Output (JSONL):
-    {"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."}
-
-Examples:
-    # Extract specific snapshot
-    archivebox extract 01234567-89ab-cdef-0123-456789abcdef
-
-    # Pipe from snapshot command
-    archivebox snapshot https://example.com | archivebox extract
-
-    # Run specific plugins only
-    archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef
-
-    # Chain commands
-    archivebox crawl https://example.com | archivebox snapshot | archivebox extract
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox extract"
-
-import sys
-from collections import defaultdict
-
-import rich_click as click
-
-
-def process_archiveresult_by_id(archiveresult_id: str) -> int:
-    """
-    Re-run extraction for a single ArchiveResult by ID.
-
-    ArchiveResults are projected status rows, not queued work items. Re-running
-    a single result means resetting that row and queueing its parent snapshot
-    through the shared crawl runner with the corresponding plugin selected.
-    """
-    from rich import print as rprint
-    from django.utils import timezone
-    from archivebox.core.models import ArchiveResult
-    from archivebox.services.runner import run_crawl
-
-    try:
-        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
-    except ArchiveResult.DoesNotExist:
-        rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
-        return 1
-
-    rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
-
-    try:
-        archiveresult.reset_for_retry()
-        snapshot = archiveresult.snapshot
-        snapshot.status = snapshot.StatusChoices.QUEUED
-        snapshot.retry_at = timezone.now()
-        snapshot.save(update_fields=["status", "retry_at", "modified_at"])
-
-        crawl = snapshot.crawl
-        if crawl.status != crawl.StatusChoices.STARTED:
-            crawl.status = crawl.StatusChoices.QUEUED
-        crawl.retry_at = timezone.now()
-        crawl.save(update_fields=["status", "retry_at", "modified_at"])
-
-        run_crawl(str(crawl.id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
-        archiveresult.refresh_from_db()
-
-        if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
-            print(f"[green]Extraction succeeded: {archiveresult.output_str}[/green]")
-            return 0
-        elif archiveresult.status == ArchiveResult.StatusChoices.NORESULTS:
-            print(f"[dim]Extraction completed with no results: {archiveresult.output_str}[/dim]")
-            return 0
-        elif archiveresult.status == ArchiveResult.StatusChoices.FAILED:
-            print(f"[red]Extraction failed: {archiveresult.output_str}[/red]", file=sys.stderr)
-            return 1
-        else:
-            # Still in progress or backoff - not a failure
-            print(f"[yellow]Extraction status: {archiveresult.status}[/yellow]")
-            return 0
-
-    except Exception as e:
-        print(f"[red]Extraction error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-        return 1
-
-
-def run_plugins(
-    args: tuple,
-    records: list[dict] | None = None,
-    plugins: str = "",
-    wait: bool = True,
-    emit_results: bool = True,
-) -> int:
-    """
-    Run plugins on Snapshots from input.
-
-    Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from rich import print as rprint
-    from django.utils import timezone
-
-    from archivebox.misc.jsonl import (
-        read_args_or_stdin,
-        write_record,
-        TYPE_SNAPSHOT,
-        TYPE_ARCHIVERESULT,
-    )
-    from archivebox.core.models import Snapshot
-    from archivebox.services.runner import run_crawl
-
-    is_tty = sys.stdout.isatty()
-
-    # Parse comma-separated plugins list once (reused in creation and filtering)
-    plugins_list = [p.strip() for p in plugins.split(",") if p.strip()] if plugins else []
-
-    # Parse stdin/args exactly once per CLI invocation.
-    # `main()` may already have consumed stdin to distinguish Snapshot input from
-    # ArchiveResult IDs; if so, it must pass the parsed records through here
-    # instead of asking this helper to reread an already-drained pipe.
-    if records is None:
-        records = list(read_args_or_stdin(args))
-
-    if not records:
-        rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
-        return 1
-
-    # Gather snapshot IDs and optional plugin constraints to process
-    snapshot_ids = set()
-    requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set)
-    for record in records:
-        record_type = record.get("type")
-
-        if record_type == TYPE_SNAPSHOT:
-            snapshot_id = record.get("id")
-            if snapshot_id:
-                snapshot_ids.add(snapshot_id)
-            elif record.get("url"):
-                # Look up by URL (get most recent if multiple exist)
-                snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
-                if snap:
-                    snapshot_ids.add(str(snap.id))
-                else:
-                    rprint(f"[yellow]Snapshot not found for URL: {record['url']}[/yellow]", file=sys.stderr)
-
-        elif record_type == TYPE_ARCHIVERESULT:
-            snapshot_id = record.get("snapshot_id")
-            if snapshot_id:
-                snapshot_ids.add(snapshot_id)
-                plugin_name = record.get("plugin")
-                if plugin_name and not plugins_list:
-                    requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
-
-        elif "id" in record:
-            # Assume it's a snapshot ID
-            snapshot_ids.add(record["id"])
-
-    if not snapshot_ids:
-        rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
-        return 1
-
-    # Get snapshots and ensure they have pending ArchiveResults
-    processed_count = 0
-    for snapshot_id in snapshot_ids:
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-        except Snapshot.DoesNotExist:
-            rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
-            continue
-
-        requested_plugin_names = set(plugins_list) | requested_plugins_by_snapshot.get(str(snapshot.id), set())
-        for plugin_name in requested_plugin_names:
-            existing_result = snapshot.archiveresult_set.filter(plugin=plugin_name).order_by("-created_at").first()
-            if existing_result:
-                existing_result.reset_for_retry()
-
-        # Reset snapshot status to allow processing
-        if snapshot.status == Snapshot.StatusChoices.SEALED:
-            snapshot.status = Snapshot.StatusChoices.STARTED
-            snapshot.retry_at = timezone.now()
-            snapshot.save()
-
-        processed_count += 1
-
-    if processed_count == 0:
-        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
-        return 1
-
-    rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
-
-    # Run orchestrator if --wait (default)
-    if wait:
-        rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
-        snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id in snapshot_ids:
-            try:
-                snapshot = Snapshot.objects.only("id", "crawl_id").get(id=snapshot_id)
-            except Snapshot.DoesNotExist:
-                continue
-            snapshot_ids_by_crawl[str(snapshot.crawl_id)].add(str(snapshot.id))
-
-        for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
-            selected_plugins = (
-                plugins_list
-                or sorted(
-                    {plugin for snapshot_id in crawl_snapshot_ids for plugin in requested_plugins_by_snapshot.get(str(snapshot_id), set())},
-                )
-                or None
-            )
-            run_crawl(
-                crawl_id,
-                snapshot_ids=sorted(crawl_snapshot_ids),
-                selected_plugins=selected_plugins,
-            )
-
-    if not emit_results:
-        return 0
-
-    # Output results as JSONL (when piped) or human-readable (when TTY)
-    for snapshot_id in snapshot_ids:
-        try:
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-            results = snapshot.archiveresult_set.all()
-            if plugins_list:
-                results = results.filter(plugin__in=plugins_list)
-
-            for result in results:
-                if is_tty:
-                    status_color = {
-                        "succeeded": "green",
-                        "failed": "red",
-                        "skipped": "yellow",
-                    }.get(result.status, "dim")
-                    rprint(
-                        f"  [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ''}",
-                        file=sys.stderr,
-                    )
-                else:
-                    write_record(result.to_json())
-        except Snapshot.DoesNotExist:
-            continue
-
-    return 0
-
-
-def is_archiveresult_id(value: str) -> bool:
-    """Check if value looks like an ArchiveResult UUID."""
-    import re
-
-    uuid_pattern = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
-    if not uuid_pattern.match(value):
-        return False
-    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
-    from archivebox.core.models import ArchiveResult
-
-    return ArchiveResult.objects.filter(id=value).exists()
-
-
-@click.command()
-@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run (e.g., screenshot,singlefile)")
-@click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
-@click.argument("args", nargs=-1)
-def main(plugins: str, wait: bool, args: tuple):
-    """Run plugins on Snapshots, or process existing ArchiveResults by ID"""
-    from archivebox.misc.jsonl import read_args_or_stdin
-
-    # Read all input
-    records = list(read_args_or_stdin(args))
-
-    if not records:
-        from rich import print as rprint
-
-        rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
-        sys.exit(1)
-
-    # Check if input looks like existing ArchiveResult IDs to process
-    all_are_archiveresult_ids = all(is_archiveresult_id(r.get("id") or r.get("url", "")) for r in records)
-
-    if all_are_archiveresult_ids:
-        # Process existing ArchiveResults by ID
-        from rich import print as rprint
-
-        exit_code = 0
-        for record in records:
-            archiveresult_id = record.get("id") or record.get("url")
-            if not isinstance(archiveresult_id, str):
-                rprint(f"[red]Invalid ArchiveResult input: {record}[/red]", file=sys.stderr)
-                exit_code = 1
-                continue
-            result = process_archiveresult_by_id(archiveresult_id)
-            if result != 0:
-                exit_code = result
-        sys.exit(exit_code)
-    else:
-        # Default behavior: run plugins on Snapshots from input
-        sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,133 +1,32 @@
 #!/usr/bin/env python3
-__package__ = "archivebox.cli"
-__command__ = "archivebox help"

-import os
-from pathlib import Path
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox help'

-import click
-from rich import print
-from rich.panel import Panel
+import sys
+import argparse
+
+from typing import Optional, List, IO
+
+from ..main import help
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin


-def help() -> None:
-    """Print the ArchiveBox help message and usage"""
-
-    from archivebox.cli import ArchiveBoxGroup
-    from archivebox.config import CONSTANTS
-    from archivebox.config.common import get_config
-    from archivebox.config.permissions import IN_DOCKER
-    from archivebox.misc.logging_util import log_cli_command
-
-    log_cli_command("help", [], None, ".")
-
-    COMMANDS_HELP_TEXT = (
-        "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.meta_commands.keys()
-        )
-        + "\n\n    "
-        + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.setup_commands.keys()
-        )
-        + "\n\n    "
-        + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}" for cmd in ArchiveBoxGroup.archive_commands.keys()
-        )
+@docstring(help.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=help.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
    )
-
-    DOCKER_USAGE = (
-        """
-[dodger_blue3]Docker Usage:[/dodger_blue3]
-    [grey53]# using Docker Compose:[/grey53]
-    [blue]docker compose run[/blue] [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-
-    [grey53]# using Docker:[/grey53]
-    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 8000:8000[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-"""
-        if IN_DOCKER
-        else ""
-    )
-    DOCKER_DOCS = (
-        "\n    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker[/link]"
-        if IN_DOCKER
-        else ""
-    )
-    DOCKER_OUTSIDE_HINT = "\n    [grey53]# outside of Docker:[/grey53]" if IN_DOCKER else ""
-    DOCKER_CMD_PREFIX = "[blue]docker ... [/blue]" if IN_DOCKER else ""
-
-    print(f"""{DOCKER_USAGE}
-[deep_sky_blue4]Usage:[/deep_sky_blue4]{DOCKER_OUTSIDE_HINT}
-    [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-
-[deep_sky_blue4]Commands:[/deep_sky_blue4]
-    {COMMANDS_HELP_TEXT}
-
-[deep_sky_blue4]Documentation:[/deep_sky_blue4]
-    [link=https://github.com/ArchiveBox/ArchiveBox/wiki]https://github.com/ArchiveBox/ArchiveBox/wiki[/link]{DOCKER_DOCS}
-    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage[/link]
-    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
-""")
-
-    config = get_config()
-    if os.access(config.ARCHIVE_DIR, os.R_OK) and config.ARCHIVE_DIR.is_dir():
-        pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
-        EXAMPLE_USAGE = f"""
-[light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]
-
-[violet]Hint:[/violet] [i]Common maintenance tasks:[/i]
-    [dark_green]archivebox[/dark_green] [green]init[/green]      [grey53]# make sure database is up-to-date (safe to run multiple times)[/grey53]
-    [dark_green]archivebox[/dark_green] [green]install[/green]   [grey53]# make sure plugins are up-to-date (wget, chrome, singlefile, etc.)[/grey53]
-    [dark_green]archivebox[/dark_green] [green]status[/green]    [grey53]# get a health checkup report on your collection[/grey53]
-    [dark_green]archivebox[/dark_green] [green]update[/green]    [grey53]# retry any previously failed or interrupted archiving tasks[/grey53]
-
-[violet]Hint:[/violet] [i]More example usage:[/i]
-    [dark_green]archivebox[/dark_green] [green]add[/green] --depth=1 "https://example.com/some/page"
-    [dark_green]archivebox[/dark_green] [green]list[/green] --sort=timestamp --csv=timestamp,downloaded_at,url,title
-    [dark_green]archivebox[/dark_green] [green]schedule[/green] --every=day --depth=1 "https://example.com/some/feed.rss"
-    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:8000[/blue]                [grey53]# Start the Web UI / API server[/grey53]
-"""
-        print(
-            Panel(
-                EXAMPLE_USAGE,
-                expand=False,
-                border_style="grey53",
-                title="[green3]:white_check_mark: A collection [light_slate_blue]DATA DIR[/light_slate_blue] is currently active[/green3]",
-                subtitle="Commands run inside this dir will only apply to this collection.",
-            ),
-        )
-    else:
-        DATA_SETUP_HELP = "\n"
-        if IN_DOCKER:
-            DATA_SETUP_HELP += "[violet]Hint:[/violet] When using Docker, you need to mount a volume to use as your data dir:\n"
-            DATA_SETUP_HELP += "    docker run [violet]-v /some/path/data:/data[/violet] archivebox/archivebox ...\n\n"
-        DATA_SETUP_HELP += "To load an [dark_blue]existing[/dark_blue] collection:\n"
-        DATA_SETUP_HELP += "    1. [green]cd[/green] ~/archivebox/data     [grey53]# go into existing [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
-        DATA_SETUP_HELP += f"    2. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# migrate to latest version (safe to run multiple times)[/grey53]\n"
-        DATA_SETUP_HELP += f"    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-update all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
-        DATA_SETUP_HELP += f"    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ...get help with next steps... [/grey53]\n\n"
-        DATA_SETUP_HELP += "To start a [sea_green1]new[/sea_green1] collection:\n"
-        DATA_SETUP_HELP += "    1. [green]mkdir[/green] ~/archivebox/data  [grey53]# create a new, empty [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
-        DATA_SETUP_HELP += "    2. [green]cd[/green] ~/archivebox/data     [grey53]# cd into the new directory[/grey53]\n"
-        DATA_SETUP_HELP += f"    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# initialize ArchiveBox in the new data dir[/grey53]\n"
-        DATA_SETUP_HELP += f"    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-install all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
-        DATA_SETUP_HELP += f"    5. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ... get help with next steps... [/grey53]\n"
-        print(
-            Panel(
-                DATA_SETUP_HELP,
-                expand=False,
-                border_style="grey53",
-                title="[red]:cross_mark: No collection is currently active[/red]",
-                subtitle="All archivebox [green]commands[/green] should be run from inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
-            ),
-        )
+    parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)
+    
+    help(out_dir=pwd or OUTPUT_DIR)


-@click.command()
-@click.option("--help", "-h", is_flag=True, help="Show help")
-def main(**kwargs):
-    """Print the ArchiveBox help message and usage"""
-    return help()
-
-
-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,185 +1,52 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox init'

-import os
 import sys
-from pathlib import Path
+import argparse

-from rich import print
-import rich_click as click
+from typing import Optional, List, IO

-from archivebox.misc.util import docstring, enforce_types
+from ..main import init
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin


-def _display_data_path(path: Path, data_dir: Path) -> str:
-    path = Path(path).resolve()
-    data_dir = Path(data_dir).resolve()
-    try:
-        return f"./{path.relative_to(data_dir)}"
-    except ValueError:
-        return str(path)
-
-
-@enforce_types
-def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
-    """Initialize a new ArchiveBox collection in the current directory"""
-
-    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
-    from archivebox.config.common import get_config
-    from archivebox.config.collection import write_config_file
-    from archivebox.misc.db import apply_migrations
-
-    config = get_config()
-
-    # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
-    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
-    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
-
-    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
-    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
-    if is_empty and not existing_index:
-        print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
-        print("[green]----------------------------------------------------------------------[/green]")
-    elif existing_index:
-        # TODO: properly detect and print the existing version in current index as well
-        print(f"[green][*] Verifying and updating existing ArchiveBox collection to v{VERSION}...[/green]")
-        print("[green]----------------------------------------------------------------------[/green]")
-    else:
-        if force:
-            print("[red][!] This folder appears to already have files in it, but no index.sqlite3 is present.[/red]")
-            print("[red]    Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).[/red]")
-        else:
-            print(
-                "[red][X] This folder appears to already have files in it, but no index.sqlite3 present.[/red]\n\n"
-                "    You must run init in a completely empty directory, or an existing data folder.\n\n"
-                "    [violet]Hint:[/violet] To import an existing data folder make sure to cd into the folder first, \n"
-                "    then run and run 'archivebox init' to pick up where you left off.\n\n"
-                "    (Always make sure your data folder is backed up first before updating ArchiveBox)",
-            )
-            raise SystemExit(2)
-
-    if existing_index:
-        print("\n[green][*] Verifying archive folder structure...[/green]")
-    else:
-        print("\n[green][+] Building archive folder structure...[/green]")
-
-    archive_path = _display_data_path(config.ARCHIVE_DIR, DATA_DIR)
-    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, DATA_DIR)
-    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, DATA_DIR)
-    print(f"    + {archive_path}, {sources_path}, {logs_path}...")
-    Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
-    config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
-    config.USERS_DIR.mkdir(parents=True, exist_ok=True)
-    Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
-
-    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...")
-
-    # create the .archivebox_id file with a unique ID for this collection
-    from archivebox.config.paths import _get_collection_id
-
-    _get_collection_id(DATA_DIR, force_create=True)
-
-    # create the ArchiveBox.conf file
-    write_config_file({"SECRET_KEY": config.SECRET_KEY})
-
-    if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
-        print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]")
-    else:
-        print("\n[green][+] Building main SQL index and running initial migrations...[/green]")
-
-    from archivebox.config.django import setup_django
-
-    setup_django()
-
-    for migration_line in apply_migrations(DATA_DIR):
-        sys.stdout.write(f"    {migration_line}\n")
-
-    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
-    print()
-    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, DATA_DIR)}")
-
-    # from django.contrib.auth.models import User
-    #     call_command("createsuperuser", interactive=True)
-
-    print()
-    print("[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]")
-
-    from archivebox.core.models import Snapshot
-
-    snapshot_count = 0
-
-    if existing_index:
-        snapshot_count = Snapshot.objects.count()
-        print(f"    √ Loaded {snapshot_count} links from existing main index.")
-
-    print("    > Skipping orphan snapshot import during init.")
-    print()
-    print("    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
-    print("        archivebox update")
-
-    print("\n[green]----------------------------------------------------------------------[/green]")
-
-    from django.contrib.auth.models import User
-
-    config = get_config()
-    if (config.ADMIN_USERNAME and config.ADMIN_PASSWORD) and not User.objects.filter(
-        username=config.ADMIN_USERNAME,
-    ).exists():
-        print("[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]")
-        User.objects.create_superuser(username=config.ADMIN_USERNAME, password=config.ADMIN_PASSWORD)
-
-    if existing_index:
-        print("[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]")
-    else:
-        print(f"[green][√] Done. A new ArchiveBox collection was initialized ({snapshot_count} links).[/green]")
-
-    CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
-    CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
-
-    from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir
-
-    config = get_config()
-    config.TMP_DIR.mkdir(parents=True, exist_ok=True)
-    config.LIB_DIR.mkdir(parents=True, exist_ok=True)
-    (config.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
-
-    working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
-    if working_tmp_dir:
-        working_tmp_dir.mkdir(parents=True, exist_ok=True)
-
-    working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
-    if working_lib_dir:
-        working_lib_dir.mkdir(parents=True, exist_ok=True)
-        (working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)
-
-    if install:
-        from archivebox.cli.archivebox_install import install as install_method
-
-        install_method()
-
-    if Snapshot.objects.count() < 25:  # hide the hints for experienced users
-        print()
-        print("    [violet]Hint:[/violet] To view your archive index, run:")
-        print(
-            "        archivebox server  # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]",
-        )
-        print()
-        print("    To add new links, you can run:")
-        print("        archivebox add < ~/some/path/to/list_of_links.txt")
-        print()
-        print("    For more usage and examples, run:")
-        print("        archivebox help")
-
-
-@click.command()
-@click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
-@click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
-@click.option("--install", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
@docstring(init.__doc__)
-def main(**kwargs) -> None:
-    init(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=init.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--force', # '-f',
+        action='store_true',
+        help='Ignore unrecognized files in current directory and initialize anyway',
+    )
+    parser.add_argument(
+        '--quick', '-q',
+        action='store_true',
+        help='Run any updates or migrations without rechecking all snapshot dirs',
+    )
+    parser.add_argument(
+        '--setup', #'-s',
+        action='store_true',
+        help='Automatically install dependencies and extras used for archiving',
+    )
+    command = parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)

+    init(
+        force=command.force,
+        quick=command.quick,
+        setup=command.setup,
+        out_dir=pwd or OUTPUT_DIR,
+    )
+    

-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,106 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-
-import os
-
-import rich_click as click
-from rich import print
-
-from archivebox.misc.util import docstring, enforce_types
-
-
-@enforce_types
-def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
-    """Detect and install ArchiveBox dependencies by running the abx-dl install flow
-
-    Examples:
-        archivebox install                              # Install all dependencies
-        archivebox install wget curl                    # Install only wget and curl
-        archivebox install --binproviders=pip yt-dlp    # Install yt-dlp using only pip
-        archivebox install --binproviders=brew,apt      # Install all deps using only brew or apt
-    """
-
-    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
-    from archivebox.config.common import get_config
-    from archivebox.misc.logging import stderr
-    from archivebox.cli.archivebox_init import init
-
-    config = get_config()
-    archive_dir = config.ARCHIVE_DIR
-    if not (os.access(archive_dir, os.R_OK) and archive_dir.is_dir()):
-        init()  # must init full index because we need a db to store Binary entries in
-
-    # Show what we're installing
-    if binaries:
-        print(f"\n[green][+] Installing specific binaries: {', '.join(binaries)}[/green]")
-    else:
-        print("\n[green][+] Detecting and installing all ArchiveBox dependencies...[/green]")
-
-    if binproviders != "*":
-        print(f"[green][+] Using providers: {binproviders}[/green]")
-
-    if IS_ROOT:
-        EUID = os.geteuid()
-        print()
-        print(f"[yellow]:warning:  Running as UID=[blue]{EUID}[/blue].[/yellow]")
-        print(f"    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].")
-        print()
-
-    if dry_run:
-        print("[dim]Dry run - would run the abx-dl install flow[/dim]")
-        return
-
-    # Set up Django
-    from archivebox.config.django import setup_django
-
-    setup_django()
-
-    plugin_names = list(binaries)
-    if binproviders != "*":
-        plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
-
-    print("[+] Running installer via abx-dl bus...")
-    print()
-
-    from archivebox.services.runner import run_install
-
-    run_install(plugin_names=plugin_names or None)
-
-    print()
-
-    # Check for superuser
-    from django.contrib.auth import get_user_model
-
-    User = get_user_model()
-
-    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
-        stderr("\n[+] Don't forget to create a new admin user for the Web UI...", color="green")
-        stderr("    archivebox manage createsuperuser")
-
-    print()
-
-    # Show version to display full status including installed binaries
-    # Django is already loaded, so just import and call the function directly
-    from archivebox.cli.archivebox_version import version as show_version
-
-    show_version(quiet=False)
-
-
-@click.command()
-@click.argument("binaries", nargs=-1, type=str, required=False)
-@click.option(
-    "--binproviders",
-    "-p",
-    default="*",
-    help="Comma-separated list of providers to use (pip,npm,brew,apt,env,custom) or * for all",
-    show_default=True,
-)
-@click.option("--dry-run", "-d", is_flag=True, help="Show what would happen without actually running", default=False)
-@docstring(install.__doc__)
-def main(**kwargs) -> None:
-    install(**kwargs)
-
-
-if __name__ == "__main__":
-    main()
@@ -1,57 +1,139 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
-__command__ = "archivebox list"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox list'

 import sys
+import argparse

-import rich_click as click
+from typing import Optional, List, IO

-from archivebox.cli.archivebox_snapshot import list_snapshots
+from ..main import list_all
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..index import (
+    LINK_FILTERS,
+    get_indexed_folders,
+    get_archived_folders,
+    get_unarchived_folders,
+    get_present_folders,
+    get_valid_folders,
+    get_invalid_folders,
+    get_duplicate_folders,
+    get_orphaned_folders,
+    get_corrupted_folders,
+    get_unrecognized_folders,
+)
+from ..logging_util import SmartFormatter, reject_stdin, stderr


-@click.command()
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def main(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-) -> None:
-    """List Snapshots."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
+@docstring(list_all.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=list_all.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--csv', #'-c',
+        type=str,
+        help="Print the output in CSV format with the given columns, e.g.: timestamp,url,extension",
+        default=None,
+    )
+    group.add_argument(
+        '--json', #'-j',
+        action='store_true',
+        help="Print the output in JSON format with all columns included",
+    )
+    group.add_argument(
+        '--html',
+        action='store_true',
+        help="Print the output in HTML format"
+    )
+    parser.add_argument(
+        '--with-headers',
+        action='store_true',
+        help='Include the headers in the output document' 
+    )
+    parser.add_argument(
+        '--sort', #'-s',
+        type=str,
+        help="List the links sorted using the given key, e.g. timestamp or updated",
+        default=None,
+    )
+    parser.add_argument(
+        '--before', #'-b',
+        type=float,
+        help="List only links bookmarked before (less than) the given timestamp",
+        default=None,
+    )
+    parser.add_argument(
+        '--after', #'-a',
+        type=float,
+        help="List only links bookmarked after (greater than or equal to) the given timestamp",
+        default=None,
+    )
+    parser.add_argument(
+        '--status',
+        type=str,
+        choices=('indexed', 'archived', 'unarchived', 'present', 'valid', 'invalid', 'duplicate', 'orphaned', 'corrupted', 'unrecognized'),
+        default='indexed',
+        help=(
+            'List only links or data directories that have the given status\n'
+            f'    indexed       {get_indexed_folders.__doc__} (the default)\n'
+            f'    archived      {get_archived_folders.__doc__}\n'
+            f'    unarchived    {get_unarchived_folders.__doc__}\n'
+            '\n'
+            f'    present       {get_present_folders.__doc__}\n'
+            f'    valid         {get_valid_folders.__doc__}\n'
+            f'    invalid       {get_invalid_folders.__doc__}\n'
+            '\n'
+            f'    duplicate     {get_duplicate_folders.__doc__}\n'
+            f'    orphaned      {get_orphaned_folders.__doc__}\n'
+            f'    corrupted     {get_corrupted_folders.__doc__}\n'
+            f'    unrecognized  {get_unrecognized_folders.__doc__}\n'
+        )
+    )
+    parser.add_argument(
+        '--filter-type', '-t',
+        type=str,
+        choices=(*LINK_FILTERS.keys(), 'search'),
+        default='exact',
+        help='Type of pattern matching to use when filtering URLs',
+    )
+    parser.add_argument(
+        'filter_patterns',
+        nargs='*',
+        type=str,
+        default=None,
+        help='List only URLs matching these filter patterns'
+    )
+    command = parser.parse_args(args or ())
+    reject_stdin(stdin)

+    if command.with_headers and not (command.json or command.html or command.csv):
+        stderr(
+            '[X] --with-headers can only be used with --json, --html or --csv options\n',
+            color='red',
+        )
+        raise SystemExit(2)

-if __name__ == "__main__":
-    main()
+    matching_folders = list_all(
+        filter_patterns=command.filter_patterns,
+        filter_type=command.filter_type,
+        status=command.status,
+        after=command.after,
+        before=command.before,
+        sort=command.sort,
+        csv=command.csv,
+        json=command.json,
+        html=command.html,
+        with_headers=command.with_headers,
+        out_dir=pwd or OUTPUT_DIR,
+    )
+    raise SystemExit(not matching_folders)
+
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,102 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox machine <action> [--filters]
-
-Manage Machine records (system-managed, mostly read-only).
-
-Machine records track the host machines where ArchiveBox runs.
-They are created automatically by the system and are primarily for debugging.
-
-Actions:
-    list    - List Machines as JSONL (with optional filters)
-
-Examples:
-    # List all machines
-    archivebox machine list
-
-    # List machines by hostname
-    archivebox machine list --hostname__icontains=myserver
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox machine"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_utils import apply_filters
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_machines(
-    hostname__icontains: str | None = None,
-    os_platform: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Machines as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Machine
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Machine.objects.all().order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "hostname__icontains": hostname__icontains,
-        "os_platform": os_platform,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for machine in queryset:
-        if is_tty:
-            rprint(f"[cyan]{machine.hostname:30}[/cyan] [dim]{machine.os_platform:10}[/dim] {machine.id}")
-        else:
-            write_record(machine.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} machines[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Machine records (read-only, system-managed)."""
-    pass
-
-
-@main.command("list")
-@click.option("--hostname__icontains", help="Filter by hostname contains")
-@click.option("--os-platform", help="Filter by OS platform")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(hostname__icontains: str | None, os_platform: str | None, limit: int | None):
-    """List Machines as JSONL."""
-    sys.exit(
-        list_machines(
-            hostname__icontains=hostname__icontains,
-            os_platform=os_platform,
-            limit=limit,
-        ),
-    )
-
-
-if __name__ == "__main__":
-    main()
Author	SHA1	Message	Date
Nick Sweeting	726bcabd82	fix iterator missing chunk_size	2024-08-26 22:18:10 -07:00
Nick Sweeting	d18418cc22	fix ArchiveResult.output_summary admin formatting	2024-08-26 22:17:57 -07:00