fix iterator missing chunk_size

fix ArchiveResult.output_summary admin formatting
2024-08-26 22:18:10 -07:00 · 2024-08-26 22:17:57 -07:00
468 changed files with 14539 additions and 23633 deletions
@@ -9,7 +9,6 @@ __pycache__/
 .pdm-python
 .eggs/
 .git/
-.vscode/
 !.git/HEAD
 !.git/refs/heads/*

@@ -32,8 +31,6 @@ deb_dist/
 pip_dist/
 assets/
 docker/
-website/
-typings/

 data/
 data*/
@@ -0,0 +1,2 @@
+**/*.lock
+**/*-lock.json
@@ -1,26 +0,0 @@
-# Read the Docs config for https://docs.archivebox.io
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
-
-version: 2
-
-submodules:
-  include: all
-  recursive: true
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.12"
-    #nodejs: "20"     # not needed unless we need the full archivebox to run while building docs for some reason
-
-sphinx:
-  configuration: docs/conf.py
-
-formats:
-  - pdf
-  - epub
-
-# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-python:
-  install:
-    - requirements: docs/requirements.txt
@@ -1,198 +0,0 @@
-name: 🐞 Bug report
-description: Report a bug or error you encountered in ArchiveBox
-title: "Bug: ..."
-assignees:
-  - pirate
-type: 'Bug'
-body:
-  - type: markdown
-    attributes:
-      value: |
-        *Please note:* it is normal to see errors occasionally for some extractors on some URLs (not every extractor will work on every type of page).  
-        Please report archiving errors if you are seeing them *consistently across many URLs* or if they are *preventing you from using ArchiveBox*.
-
-  - type: textarea
-    id: description
-    attributes:
-      label: Provide a screenshot and describe the bug
-      description: |
-        Attach a screenshot and describe what the issue is, what you expected to happen, and if relevant, the *URLs you were trying to archive*.  
-      placeholder: |
-        Got a bunch of 'singlefile was unable to archive this page' errors when trying to archive URLs from this site: https://example.com/xyz ...
-        I also tried to archive the same URLs using `singlefile` directly and some of them worked but not all of them. etc. ...
-    validations:
-      required: true
-
-  - type: textarea
-    id: steps_to_reproduce
-    attributes:
-      label: Steps to reproduce
-      description: Please include the exact steps you took to trigger the issue.
-      render: markdown
-      placeholder: |
-        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:latest`
-        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
-        3. Typed 'https://example.com/xyz' into the 'Add URL' input field
-        4. Clicked the 'Add+' button
-        5. Got a 500 error and saw the errors below in terminal
-    validations:
-      required: true
-
-  - type: textarea
-    id: logs
-    attributes:
-      label: Logs or errors
-      description: "Paste any terminal output, logs, or errors (check `data/logs/errors.log` as well)."
-      placeholder: |
-        ╭─────────────────────────────────────────────────────────────────────────────────────────────────────────╮
-        │ [2024-11-02 19:54:28] ArchiveBox v0.8.6rc0: archivebox add https://example.com#1234567                  │
-        ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯
-        
-        [+] [2024-11-02 19:54:29] Adding 1 links to index (crawl depth=0)...
-            > Saved verbatim input to sources/1730577269-import.txt
-            > Parsed 1 URLs from input (Generic TXT)
-            ...
-      render: shell
-    validations:
-      required: false
-
-  - type: textarea
-    id: version
-    attributes:
-      label: ArchiveBox Version
-      description: |
-        **REQUIRED:** Run the `archivebox version` command inside your collection dir and paste the *full output* here (*not just the version number*).  
-        For Docker Compose run: `docker compose run archivebox version`  
-        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
-      render: shell
-      placeholder: |
-        0.8.6
-        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
-        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
-        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
-        
-        Binary Dependencies:
-        √  python     3.11.9     venv_pip   ~/.venv/bin/python
-        √  django     5.1.2      venv_pip   ~/.venv/lib/python3.11/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.11/site-packages/django/db/backends/sqlite3/base.py
-        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
-        ...
-    validations:
-      required: true
-
-  - type: dropdown
-    id: install_method
-    validations:
-      required: true
-    attributes:
-      label: How did you install the version of ArchiveBox you are using?
-      multiple: false
-      options:
-        - pip
-        - apt
-        - brew
-        - nix
-        - Docker (or other container system like podman/LXC/Kubernetes or TrueNAS/Cloudron/YunoHost/etc.)
-        - Other
-
-  - type: dropdown
-    id: operating_system
-    validations:
-      required: true
-    attributes:
-      label: What operating system are you running on?
-      description: |
-        Please note we are *unable to provide support for Windows users* unless you are using [Docker on Windows](https://github.com/ArchiveBox/archivebox#:~:text=windows%20without%20docker).
-      multiple: false
-      options:
-        - Linux (Ubuntu/Debian/Arch/Alpine/etc.)
-        - macOS (including Docker on macOS)
-        - BSD (FreeBSD/OpenBSD/NetBSD/etc.)
-        - Windows (including WSL, WSL2, Docker Desktop on Windows)
-        - Other
-
-  - type: checkboxes
-    id: filesystem
-    attributes:
-      label: What type of drive are you using to store your ArchiveBox data?
-      description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` or `data/archive` (e.g. NFS/SMB/CIFS/etc. or FUSE/RClone/S3/B2/OneDrive/etc.)?
-      options:
-        - label: "`data/` is on a local SSD or NVMe drive"
-          required: false
-        - label: "`data/` is on a spinning hard drive or external USB drive"
-          required: false
-        - label: "`data/` is on a network mount (e.g. NFS/SMB/CIFS/etc.)"
-          required: false
-        - label: "`data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/OneDrive, etc.)"
-          required: false
-  
-
-  - type: textarea
-    id: docker_compose_yml
-    attributes:
-      label: Docker Compose Configuration
-      description: "If using Docker Compose, please share your full `docker-compose.yml` file. If using plain Docker, paste the `docker run ...` command you use."
-      placeholder: |
-        services:
-            archivebox:
-                image: archivebox/archivebox:latest
-                ports:
-                    - 8000:8000
-                volumes:
-                    - ./data:/data
-                environment:
-                    - ADMIN_USERNAME=admin
-                    - ADMIN_PASSWORD=****<redact any passwords>****
-                    - ALLOWED_HOSTS=*
-                    - CSRF_TRUSTED_ORIGINS=https://archivebox.example.com
-                    - PUBLIC_INDEX=True
-                    - PUBLIC_SNAPSHOTS=True
-                    - PUBLIC_ADD_VIEW=False
-                    ...
-
-            archivebox_scheduler:
-                image: archivebox/archivebox:latest
-                command: schedule --foreground --update --every=day
-                environment:
-                    ...
-      
-          ...
-      render: shell
-    validations:
-      required: false
-
-  - type: textarea
-    id: configuration
-    attributes:
-      label: ArchiveBox Configuration
-      description: "Please share your full `data/ArchiveBox.conf` file here."
-      render: shell
-      placeholder: |
-        [SERVER_CONFIG]
-        SECRET_KEY = "*********<redact any secrets/passwords>************"
-
-        WGET_RESTRICT_FILE_NAMES=windows
-        USE_SYSTEM_WGET=true
-        CHECK_SSL_VALIDITY=false
-        ...
-    validations:
-      required: false
-
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying anything suggested here, **we are not responsible for data loss**.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
-          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
-          - 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
-          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
-        
@@ -1,127 +0,0 @@
-name: 💡 Feature or enhancement request
-description: Suggest an idea or improvement for this project
-title: "Feature Request: ..."
-assignees:
-  - pirate
-type: 'Enhancement'
-labels: 'status: idea phase'
-body:
-  - type: dropdown
-    id: suggestion_type
-    validations:
-      required: true
-    attributes:
-      label: "What type of suggestion are you making?"
-      multiple: false
-      options:
-        - New extractor / type of content to save
-        - Proposing a new feature
-        - Modification of existing behavior
-        - Web UI or UX design improvement
-
-  - type: textarea
-    id: current_problem
-    attributes:
-      label: "What is the problem that your feature request solves?"
-      description: |
-        Describe the problem that your feature request solves, feel free to include any screenshots or examples.
-      placeholder: |
-        e.g. I need to be able to archive spanish and french subtitle files from a particular movie site https://example.com/somevideos that's going down soon.
-    validations:
-      required: true
-
-  - type: textarea
-    id: proposed_solution
-    attributes:
-      label: "What is your proposed solution?"
-      description: |
-        Describe the ideal specific solution you'd want, and whether it fits into any broader scope of changes.
-      placeholder: |
-        e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
-        The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
-    validations:
-      required: true
-
-  - type: textarea
-    id: workarounds_tried
-    attributes:
-      label: "What hacks or alternative solutions have you tried to solve the problem?"
-      description: |
-        A clear and concise description of any alternative solutions, workarounds, or other software you've considered using to fix the problem.
-      placeholder: |
-        e.g. I wait for archivebox to finish archiving the page, then I manually run `yt-dlp --subs <URL>` inside 
-        the `data/archive/<timestamp>/` directory to download the subtitle files and add them to the snapshot folder.
-    validations:
-      required: true
-
-  - type: textarea
-    id: version
-    attributes:
-      label: What version of ArchiveBox are you currently using?
-      description: |
-        We need to know what version of ArchiveBox and what OS you're currently using in order to contextualize your feature request.  
-        Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in specific environments/versions.
-          
-        Run the `archivebox version` command inside your current collection dir and paste the *full output* here (*not just the version number*).  
-        For Docker Compose run: `docker compose run archivebox version`  
-        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
-      render: shell
-      placeholder: |
-        0.8.6
-        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
-        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
-        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
-        
-        Binary Dependencies:
-        √  python     3.11.9     venv_pip   ~/.venv/bin/python
-        √  django     5.1.2      venv_pip   ~/.venv/lib/python3.11/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.11/site-packages/django/db/backends/sqlite3/base.py
-        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
-        ...
-    validations:
-      required: true
-
-  - type: checkboxes
-    id: priority
-    attributes:
-      label: "How badly do you want this new feature?"
-      options:
-        - label: "It's an urgent deal-breaker, I can't live without it"
-          required: false
-        - label: "It's important to add it in the near-mid term future"
-          required: false
-        - label: "It would be nice to have eventually"
-          required: false
-        - label: "I'm willing to [work on a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
-          required: false
-        - label: "I have [donated money](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations) to go towards fixing this issue"
-          required: false
-  
-  - type: checkboxes
-    id: satisfaction_survey
-    attributes:
-      label: Mini Survey
-      description: How do you like ArchiveBox so far?
-      options:
-        - label: "I like ArchiveBox so far / would recommend it to a friend"
-          required: false
-        - label: "I've had a lot of difficulty getting ArchiveBox set up"
-          required: false
-        - label: "I would pay $10/mo for a hosted version of ArchiveBox if it had this feature"
-          required: false
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying any workarounds or BETAs suggested here, **we are not responsible for data loss**.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 📊 [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
-          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
-          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
@@ -1,52 +0,0 @@
-name: 📑 Documentation improvement
-description: Submit an idea or correction for the Wiki documentation
-title: "Documentation: ..."
-labels: 'touches: docs'
-type: 'Enhancement'
-assignees:
-  - pirate
-body:
-  - type: markdown
-    attributes:
-      value: |
-        If you prefer, you can submit a [Pull Request](https://github.com/ArchiveBox/docs) on https://github.com/ArchiveBox/docs to edit the docs directly instead.
-
-  - type: input
-    id: page_url
-    validations:
-      required: true
-    attributes:
-      label: "What is the URL of the page you'd like to see improved?"
-      placeholder: e.g. https://github.com/ArchiveBox/docs/wiki/Install
-
-  - type: input
-    id: section_title
-    validations:
-      required: true
-    attributes:
-      label: "What is the title of the relevant section?"
-      placeholder: e.g. Option B. Automatic Setup Script
-
-  - type: textarea
-    id: suggested_edit
-    attributes:
-      label: "What is the suggested edit?"
-      placeholder: |
-        e.g. Please document how to run the automatic setup script for ArchiveBox on TempleOS.  
-        Attach images, screenshots, code snippets, etc. anything you think would help.
-    validations:
-      required: true
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to address issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
-          - 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
-          - ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)
@@ -0,0 +1,46 @@
+---
+name: 🐞 Bug report
+about: Create a report to help us improve
+title: 'Bug: ...'
+labels: 'bug'
+assignees: ''
+
+---
+
+<!--
+Please fill out the following information, 
+feel free to delete sections if they're not applicable 
+or if long issue templates annoy you.
+(the only required section is the version information)
+-->
+
+#### Describe the bug
+<!--
+A description of what the bug is, 
+what you expected to happen, 
+and any relevant context about issue.
+-->
+
+#### Steps to reproduce
+<!--
+For example:
+1. Ran ArchiveBox with the following config '...'
+2. Saw this output during archiving '....'
+3. UI didn't show the thing I was expecting '....'
+-->
+
+#### Screenshots or log output
+
+<!--
+If applicable, post any relevant screenshots or copy/pasted terminal output from ArchiveBox.
+If you're reporting a parsing / importing error, **you must paste a copy of your redacted import file here**.
+-->
+
+#### ArchiveBox version
+
+<!-- Run the `archivebox version` command locally then copy paste the result here: -->
+```logs
+replace this line with the *full*, unshortened output of running `archivebox version`
+```
+<!-- Tickets without full version info will closed until it is provided,
+we need the full output here to help you solve your issue -->
@@ -1,8 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: 💬 Chat with us on Zulip
-    url: https://zulip.archivebox.io
-    about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
-  - name: 💁‍♂️ Hire us to provide archiving for your organization
-    url: https://docs.monadical.com/s/archivebox-consulting-services
-    about: "We provide hosting, develoment, and support, including on-prem/cloud setup w/ SSO & storage, CAPTCHA/rate-limiting avoidance, etc."
@@ -0,0 +1,17 @@
+---
+name: 📑 Documentation change
+about: Submit a suggestion for the Wiki documentation
+title: 'Documentation: Improvement request ...'
+labels: ''
+assignees: ''
+
+---
+<!-- If you perfer, you can make a PR to https://github.com/ArchiveBox/docs instead of opening an issue -->
+
+## Wiki Page URL
+<!-- e.g. https://github.com/pirate/ArchiveBox/wiki/Configuration#use_color -->
+
+
+## Suggested Edit
+<!-- e.g. Please add more example usages, or please fix `xyz` typo to be `abc`. -->
+
@@ -0,0 +1,50 @@
+---
+name: 💡 Feature request
+about: Suggest an idea for this project
+title: 'Feature Request: ...'
+labels: 'changes: behavior,status: idea phase'
+assignees: ''
+
+---
+
+<!--
+Please fill out the following information, 
+feel free to delete sections if they're not applicable 
+or if long issue templates annoy you :)
+-->
+
+## Type 
+
+ - [ ] General question or discussion
+ - [ ] Propose a brand new feature
+ - [ ] Request modification of existing behavior or design
+
+## What is the problem that your feature request solves
+<!--
+e.g. I need to be able to archive spanish and french subtitle files  
+from a particular <example.com> movie site that's going down soon.
+-->
+
+## Describe the ideal specific solution you'd want, and whether it fits into any broader scope of changes
+<!--
+e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
+The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
+-->
+
+## What hacks or alternative solutions have you tried to solve the problem?
+<!--
+A clear and concise description of any alternative solutions, 
+workarounds, or other software you've considered using to fix the problem.
+-->
+
+## How badly do you want this new feature?
+
+ - [ ] It's an urgent deal-breaker, I can't live without it
+ - [ ] It's important to add it in the near-mid term future
+ - [ ] It would be nice to have eventually
+
+---
+
+ - [ ] I'm willing to contribute [dev time](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) / [money](https://github.com/sponsors/pirate) to fix this issue
+ - [ ] I like ArchiveBox so far / would recommend it to a friend
+ - [ ] I've had a lot of difficulty getting ArchiveBox set up
@@ -0,0 +1,9 @@
+---
+name: 💬 Question, discussion, or support request
+about: Start a discussion or ask a question about ArchiveBox
+title: 'Question: ...'
+labels: ''
+assignees: ''
+
+---
+
@@ -81,7 +81,6 @@ jobs:
          builder: ${{ steps.buildx.outputs.name }}
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache-new
          platforms: linux/amd64,linux/arm64
@@ -1,64 +0,0 @@
-# Simple workflow for deploying static content to GitHub Pages
-name: Deploy static content to Pages
-
-on:
-  # Runs on pushes targeting the default branch
-  push:
-    branches: ["dev"]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
-# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
-concurrency:
-  group: "pages"
-  cancel-in-progress: false
-
-jobs:
-  # Single deploy job since we're just deploying
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - name: Copy README.md into place
-        run: |
-          rm -f ./website/README.md
-          cp ./README.md ./website/README.md
-      - name: Setup Pages
-        uses: actions/configure-pages@v3
-      - name: Build with Jekyll
-        uses: actions/jekyll-build-pages@v1
-        with:
-          source: ./website
-          destination: ./_site
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v2
-
-      # - name: Checkout
-      #   uses: actions/checkout@v4
-      # - name: Setup Pages
-      #   uses: actions/configure-pages@v5
-      # - name: Upload artifact
-      #   uses: actions/upload-pages-artifact@v3
-      #   with:
-      #     # Upload entire repository
-      #     path: './website'
-
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4
@@ -102,7 +102,7 @@ jobs:
        # TODO: remove this exception for windows once we get tests passing on that platform
        if: ${{ !contains(matrix.os, 'windows') }}
        run: |
-          python -m pytest -s --basetemp=tests/out --ignore=archivebox/pkgs
+          python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist

  docker_tests:
    runs-on: ubuntu-latest
@@ -11,7 +11,6 @@ venv/
 .venv/
 .docker-venv/
 node_modules/
-typings/

 # Ignore dev lockfiles (should always be built fresh)
 pdm.dev.lock
@@ -26,16 +25,12 @@ build/
 dist/

 # Data folders
-lib/
-tmp/
 data/
 data*/
 output/
 index.sqlite3
-queue.sqlite3
 *.sqlite*
 data.*
-.archivebox_id

 # vim
 *.sw?
@@ -1,3 +1,31 @@
 [submodule "docs"]
    path = docs
    url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
+
+[submodule "deb_dist"]
+    path = deb_dist
+    url = https://github.com/ArchiveBox/debian-archivebox.git
+[submodule "brew_dist"]
+	path = brew_dist
+	url = https://github.com/ArchiveBox/homebrew-archivebox.git
+[submodule "pip_dist"]
+	path = pip_dist
+	url = https://github.com/ArchiveBox/pip-archivebox.git
+[submodule "docker"]
+	path = docker
+	url = https://github.com/ArchiveBox/docker-archivebox.git
+[submodule "archivebox/vendor/base32-crockford"]
+	path = archivebox/vendor/base32-crockford
+	url = https://github.com/jbittel/base32-crockford
+[submodule "archivebox/vendor/pocket"]
+	path = archivebox/vendor/pocket
+	url = https://github.com/tapanpandita/pocket
+[submodule "archivebox/vendor/django-taggit"]
+	path = archivebox/vendor/django-taggit
+	url = https://github.com/jazzband/django-taggit
+[submodule "archivebox/vendor/python-atomicwrites"]
+	path = archivebox/vendor/python-atomicwrites
+	url = https://github.com/untitaker/python-atomicwrites
+[submodule "archivebox/vendor/pydantic-pkgr"]
+	path = archivebox/vendor/pydantic-pkgr
+	url = https://github.com/ArchiveBox/pydantic-pkgr
@@ -0,0 +1,19 @@
+tests/
+archivebox/
+archivebox.egg-info/
+build/
+dist/
+docs/
+etc/
+.github
+.gitmodules
+.dockerignore
+.flake8
+CNAME
+_config.yml
+docker-compose.yaml
+docker-compose.yml
+Dockerfile
+MANIFEST.in
+Pipfile
+setup.py
@@ -0,0 +1,33 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+submodules:
+  include: all
+  recursive: true
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    nodejs: "20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+  - pdf
+  - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt
@@ -10,15 +10,13 @@
 #     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
 # Multi-arch build:
 #     docker buildx create --use
-#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:0.7.5 -t archivebox/archivebox:dev
+#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
 #
 # Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).

-#########################################################################################

 # Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
 FROM python:3.11-slim-bookworm
-# FROM debian:bookworm-backports

 LABEL name="archivebox" \
    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -61,29 +59,32 @@ ENV TZ=UTC \

 # Version config
 ENV PYTHON_VERSION=3.11 \
-    NODE_VERSION=22
+    NODE_VERSION=20

 # User config
 ENV ARCHIVEBOX_USER="archivebox" \
    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
+    DEFAULT_PGID=911

 # Global paths
 ENV CODE_DIR=/app \
    DATA_DIR=/data \
+    GLOBAL_VENV=/venv \
    PLAYWRIGHT_BROWSERS_PATH=/browsers
-    # GLOBAL_VENV=/venv \
-    # TODO: add TMP_DIR and LIB_DIR?
+
+# Application-level paths
+ENV APP_VENV=/app/.venv \
+    NODE_MODULES=/app/node_modules

 # Build shell config
+ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 

 ######### System Environment ####################################

-# Detect ArchiveBox version number by reading pyproject.toml
-COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN grep '^version = ' "${CODE_DIR}/pyproject.toml" | awk -F'"' '{print $2}' > /VERSION.txt
+# Detect ArchiveBox version number by reading package.json
+COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
+RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt

 # Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
 RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
@@ -96,6 +97,7 @@ RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
    && echo \
+    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
    && echo \
@@ -127,30 +129,22 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
        # 1. packaging dependencies
        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils \
+        zlib1g-dev dumb-init gosu cron unzip grep \
        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        tree nano iputils-ping \
        # nano iputils-ping dnsutils htop procps jq yq
    && rm -rf /var/lib/apt/lists/*

-# Install sonic search backend
-COPY --from=archivebox/sonic:1.4.9 /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-RUN (which sonic && sonic --version) | tee -a /VERSION.txt
-
 ######### Language Environments ####################################

 # Install Python environment
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
-    # NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
    # && apt-get update -qq \
    # && apt-get install -qq -y -t bookworm-backports --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
+    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip \
    # && rm -rf /var/lib/apt/lists/* \
    # tell PDM to allow using global system python site packages
    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
@@ -168,7 +162,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T

 # Install Node environment
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing Node $NODE_VERSION environment..." \
+    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
    && apt-get update -qq \
@@ -186,7 +180,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    ) | tee -a /VERSION.txt


-
 ######### Extractor Dependencies ##################################

 # Install apt dependencies
@@ -194,7 +187,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    echo "[+] Installing APT extractor dependencies globally using apt..." \
    && apt-get update -qq \
    && apt-get install -qq -y -t bookworm-backports \
-        curl wget git ffmpeg ripgrep \
+        curl wget git yt-dlp ffmpeg ripgrep \
        # Packages we have also needed in the past:
        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
    && rm -rf /var/lib/apt/lists/* \
@@ -202,12 +195,12 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    && ( \
        which curl && curl --version | head -n1 \
        && which wget && wget --version 2>&1 | head -n1 \
+        && which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
        && which git && git --version 2>&1 | head -n1 \
        && which rg && rg --version 2>&1 | head -n1 \
        && echo -e '\n\n' \
    ) | tee -a /VERSION.txt

-
 # Install chromium browser using playwright
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
@@ -217,7 +210,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
        at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
        libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
        libxaw7 libxcomposite1 libxdamage1 libxfont2 \
-        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
+        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils xfonts-encodings \
        # xfonts-scalable xfonts-utils xserver-common xvfb \
        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
        # libxss1 dbus dbus-x11 upower \
@@ -230,7 +223,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    && rm -rf /var/lib/apt/lists/* \
    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \
+    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
    # Save version info
@@ -240,26 +233,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    ) | tee -a /VERSION.txt

 # Install Node dependencies
-ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH"
-USER $ARCHIVEBOX_USER
-WORKDIR "/home/$ARCHIVEBOX_USER/.npm"
-RUN --mount=type=cache,target=/home/$ARCHIVEBOX_USER/.npm_cache,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT,uid=$DEFAULT_PUID,gid=$DEFAULT_PGID \
-    echo "[+] Installing NPM extractor dependencies in /home/$ARCHIVEBOX_USER/.npm..." \
-    && npm config set prefix "/home/$ARCHIVEBOX_USER/.npm" \
-    && npm install --global --prefer-offline --no-fund --no-audit --cache "/home/$ARCHIVEBOX_USER/.npm_cache" \
-        "@postlight/parser@^2.2.3" \
-        "readability-extractor@github:ArchiveBox/readability-extractor" \
-        "single-file-cli@^1.1.54" \
-        "puppeteer@^23.5.0" \
-        "@puppeteer/browsers@^2.4.0"
-USER root
-RUN ( \
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
+RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
+    && npm ci --prefer-offline --no-audit --cache /root/.npm \
+    && ( \
        which node && node --version \
        && which npm && npm version \
-        && which postlight-parser \
-        && which readability-extractor && readability-extractor --version \
-        && which single-file && single-file --version \
-        && which puppeteer && puppeteer --version \
        && echo -e '\n\n' \
    ) | tee -a /VERSION.txt

@@ -272,10 +253,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
    && apt-get update -qq \
    && apt-get install -qq -y -t bookworm-backports \
-        build-essential gcc \
+        build-essential \
        libssl-dev libldap2-dev libsasl2-dev \
        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-        pipx \
    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
    # && pdm use --venv in-project \
    # && pdm run python -m ensurepip \
@@ -284,33 +264,46 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
    # && source $GLOBAL_VENV/bin/activate \
    && pip install -r requirements.txt \
    && apt-get purge -y \
-        build-essential gcc \
+        build-essential \
    && apt-get autoremove -y \
    && rm -rf /var/lib/apt/lists/*

 # Install ArchiveBox Python package from source
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
-    && pip install -e "${CODE_DIR}[all]" \
+    # && apt-get update -qq \
+    # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
+    # && apt-get install -qq -y -t bookworm-backports \
+        # build-essential  \
+    # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
+    && pip install -e "$CODE_DIR"[sonic,ldap] \
+    # save docker image size and always remove compilers / build tools after building is complete
+    # && apt-get purge -y build-essential \
+    # && apt-get autoremove -y \
    && rm -rf /var/lib/apt/lists/*

 ####################################################

 # Setup ArchiveBox runtime config
 WORKDIR "$DATA_DIR"
-RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "/tmp/archivebox" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/tmp/archivebox" \
-    && mkdir -p "/usr/share/archivebox/lib" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/usr/share/archivebox/lib"
-
-ENV GOOGLE_API_KEY=no \
+ENV IN_DOCKER=True \
+    DISPLAY=novnc:0.0 \
+    CUSTOM_TEMPLATES_DIR=/data/templates \
+    GOOGLE_API_KEY=no \
    GOOGLE_DEFAULT_CLIENT_ID=no \
    GOOGLE_DEFAULT_CLIENT_SECRET=no \
-    TMP_DIR=/tmp/archivebox \
-    LIB_DIR=/usr/share/archivebox/lib \
    ALLOWED_HOSTS=*
+    ## No need to set explicitly, these values will be autodetected by archivebox in docker:
+    # WGET_BINARY="wget" \
+    # YOUTUBEDL_BINARY="yt-dlp" \
+    # CHROME_BINARY="/usr/bin/chromium-browser" \
+    # USE_SINGLEFILE=True \
+    # SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
+    # USE_READABILITY=True \
+    # READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
+    # USE_MERCURY=True \
+    # MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"

 # Print version for nice docker finish summary
 RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
@@ -1,21 +0,0 @@
-# Example: Using ArchiveBox in your own project
-
-FROM python:3.12-slim
-
-WORKDIR /app
-
-RUN pip install archivebox==0.8.5rc50
-RUN archivebox install
-
-RUN useradd -ms /bin/bash archivebox && chown -R archivebox /app
-
-
-
-
-
-
-
-
-
-
-
@@ -1,6 +1,6 @@
 MIT License

-Copyright (c) 2024 Nick Sweeting
+Copyright (c) 2020 Nick Sweeting

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -13,7 +13,7 @@
 <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE"><img src="https://img.shields.io/badge/Open_source-MIT-green.svg?logo=git&logoColor=green"/></a> <a href="https://github.com/ArchiveBox/ArchiveBox/commits/dev"><img src="https://img.shields.io/github/last-commit/ArchiveBox/ArchiveBox.svg?logo=Sublime+Text&logoColor=green&label=Active"/></a> &nbsp; <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?logo=github&label=Stars&logoColor=blue"/></a> &nbsp; <a href="https://hub.docker.com/r/archivebox/archivebox"><img src="https://img.shields.io/docker/pulls/archivebox/archivebox.svg?label=Docker+Pulls"/></a> <a href="https://pypi.org/project/archivebox/"><img src="https://img.shields.io/pypi/dm/archivebox?label=PyPI%20Installs&color=%235f7dae"/></a> <a href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj"><img src="https://img.shields.io/chrome-web-store/users/habonpimjphpdnmcfkaockjnffodikoj?label=Chrome%20Store&color=%231973e8"/></a>

 <!--<pre lang="bash" align="left"><code style="white-space: pre-line; text-align: left" align="left">
-curl -fsSL 'https://get.archivebox.io' | bash    # (or see pip/brew/Docker instructions below)
+curl -fsSL 'https://get.archivebox.io' | sh    # (or see pip/brew/Docker instructions below)
 </code></pre>-->

 </div>
@@ -22,12 +22,12 @@ curl -fsSL 'https://get.archivebox.io' | bash    # (or see pip/brew/Docker instr

 **ArchiveBox is a powerful, self-hosted internet archiving solution to collect, save, and view websites offline.**

-Without active preservation effort, everything on the internet eventually disappears or degrades. Archive.org does a great job as a centralized service, but saved URLs have to be public, and they can't save every type of content.
+Without active preservation effort, everything on the internet eventually dissapears or degrades. Archive.org does a great job as a centralized service, but saved URLs have to be public, and they can't save every type of content.

 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>

-> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
+> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart), [macOS](#quickstart), and [Windows](#quickstart) (WSL2), or via **[Docker](#quickstart)** ⭐️.  

 *Once installed, it can be used as a [CLI tool](#usage), [self-hosted Web App](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python library](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [one-off command](#static-archive-exporting).*

@@ -99,7 +99,7 @@ archivebox init --setup
 <br/>
 <br/>
 # Option D: Or use the optional auto setup script to install it
-curl -fsSL 'https://get.archivebox.io' | bash
+curl -fsSL 'https://get.archivebox.io' | sh
 </code></pre>
 <br/>
 <sub>Open <a href="http://localhost:8000"><code>http://localhost:8000</code></a> to see your server's Web UI ➡️</sub>
@@ -130,7 +130,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
+- [**Usable as a oneshot CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/latest/modules.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://github.com/ArchiveBox/electron-archivebox)
 - [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
 - Advanced users: support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Planned: support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [autoscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
@@ -233,7 +233,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (optional, highly recommended but not required).</li>
 <li>Run the automatic setup script.
-<pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
+<pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | sh</code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>
@@ -368,7 +368,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C

 <ul>
 <li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>)</li>
-<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
+<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | sh</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
 <li>Nix: <a href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/archivebox/default.nix"><code>nix-env --install archivebox</code></a> (contributed by <a href="https://github.com/siraben"><code>@siraben</code></a>)</li>
 <li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
 <li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
@@ -790,19 +790,7 @@ ArchiveBox bundles industry-standard tools like [Google Chrome](https://github.c
 <p><em>TIP: For better security while running ArchiveBox, and to avoid polluting your host system with a bunch of sub-dependencies that you need to keep up-to-date,<strong>it is strongly recommended to use the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">⭐️ official Docker image</a></strong> which provides everything in an easy container with simple one-liner upgrades.</em></p>
 </blockquote>

-<ul>
-<li>Language: Python <code>&gt;=3.10</code></li>
-<li>Backend: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://django-ninja.dev/">Django-Ninja</a> for REST API</li>
-<li>Frontend: <a href="https://docs.djangoproject.com/en/5.1/ref/contrib/admin/">Django Admin</a> + Vanilla HTML, CSS, JS</li>
-<li>Web Server: <a href="https://www.djangoproject.com/">Django</a> + <a href="https://channels.readthedocs.io/en/latest/"><code>channels</code></a> + <a href="https://github.com/django/daphne/"><code>daphne]</code></a></li>
-<li>Database: <a href="https://docs.djangoproject.com/en/5.1/ref/databases/#sqlite-notes">Django ORM</a> saving to <a href="https://www.sqlite.org/mostdeployed.html">SQLite3</a> <code>./data/index.sqlite</code></li>
-<li>Job Queue: <a href="https://huey.readthedocs.io/">Huey</a> using <code>./data/queue.sqlite3</code> under <code>supervisord</code></li>
-<li>Build/test/lint: <a href="https://github.com/pdm-project/pdm"><code>pdm</code></a> / <code>mypy</code>+<code>pyright</code>+<code>pytest</code> / <code>ruff</code></li>
-<li>Subdependencies: <a href="https://github.com/ArchiveBox/pydantic-pkgr"><code>pydantic-pkgr</code></a> installs apt/brew/pip/npm pkgs at runtime (e.g. <code>yt-dlp</code>, <code>singlefile</code>, <code>readability</code>, <code>git</code>)</li>
-</ul>
-
-
-These optional subdependencies used for archiving sites include:
+These optional dependencies used for archiving sites include:

 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/62a02155-05d7-4f3e-8de5-75a50a145c4f" alt="archivebox --version CLI output screenshot showing dependencies installed" width="330px" align="right" style="max-width: 100%;">

@@ -1319,8 +1307,8 @@ We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wik
 ## Developers

 - [Developer Documentation](https://github.com/ArchiveBox/ArchiveBox#archivebox-development)
- [Python API](https://docs.archivebox.io/)
- [REST API](https://demo.archivebox.io/api) (alpha)
+- [Python API](https://docs.archivebox.io/en/latest/modules.html) (alpha)
+- [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (alpha)

 ## More Info

@@ -1347,11 +1335,6 @@ For low hanging fruit / easy first tickets, see: <a href="https://github.com/Arc

 **Python API Documentation:** https://docs.archivebox.io/en/dev/archivebox.html#module-archivebox.main

-**Internal Architecture Diagrams:** https://github.com/ArchiveBox/ArchiveBox/wiki/ArchiveBox-Architecture-Diagrams
-
-<img width="100%" alt="architecture diagrams" src="https://github.com/user-attachments/assets/bdf782d5-2c8f-4168-845d-6794ff5cbfec">
-
-
 ### Setup the dev environment

 <details><summary><i>Click to expand...</i></summary>
@@ -1370,12 +1353,18 @@ git pull --recurse-submodules

 ```bash
 # Install ArchiveBox + python dependencies
-pip install uv
-uv venv
-uv sync
+python3 -m venv .venv && source .venv/bin/activate && pip install -e '.[dev]'
+# or: pipenv install --dev && pipenv shell

-archivebox init
+# Install node dependencies
+npm install
+# or
 archivebox setup
+
+# Check to see if anything is missing
+archivebox --version
+# install any missing dependencies manually, or use the helper script:
+./bin/setup.sh
 ```

 #### 2. Option B: Build the docker container and use that for development instead
@@ -1412,8 +1401,6 @@ You can also run all these in Docker. For more examples see the GitHub Actions C
 archivebox config --set DEBUG=True
 # or
 archivebox server --debug ...
-# faster dev version wo/ bg workers enabled:
-daphne -b 0.0.0.0 -p 8000 archivebox.core.asgi:application
 ```

 https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
@@ -1448,12 +1435,12 @@ services:

 # or with plain Docker:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
-docker run -it -v $PWD:/data archivebox:dev init
+docker run -it -v $PWD:/data archivebox:dev init --setup

 # or with pip:
 pip install 'git+https://github.com/pirate/ArchiveBox@dev'
 npm install 'git+https://github.com/ArchiveBox/ArchiveBox.git#dev'
-archivebox install
+archivebox init --setup
 ```

 </details>
@@ -1547,9 +1534,6 @@ Extractors take the URL of a page to archive, write their output to the filesyst

 **The process to contribute a new extractor is like this:**

-> [!IMPORTANT]
-> This process is getting much easier after v0.8.x, there is a new plugin system under development: https://github.com/ArchiveBox/ArchiveBox/releases/tag/v0.8.4-rc
-
 1. [Open an issue](https://github.com/ArchiveBox/ArchiveBox/issues/new?assignees=&labels=changes%3A+behavior%2Cstatus%3A+idea+phase&template=feature_request.md&title=Feature+Request%3A+...) with your propsoed implementation (please link to the pages of any new external dependencies you plan on using)
 2. Ensure any dependencies needed are easily installable via a package managers like `apt`, `brew`, `pip3`, `npm`
   (Ideally, prefer to use external programs available via `pip3` or `npm`, however we do support using any binary installable via package manager that exposes a CLI/Python API and writes output to stdout or the filesystem.)
@@ -1578,6 +1562,8 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 # or individually:
 ./bin/build_docs.sh
 ./bin/build_pip.sh
+./bin/build_deb.sh
+./bin/build_brew.sh
 ./bin/build_docker.sh
 ```

@@ -1594,6 +1580,8 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 # or individually:
 ./bin/release_docs.sh
 ./bin/release_pip.sh
+./bin/release_deb.sh
+./bin/release_brew.sh
 ./bin/release_docker.sh
 ```

@@ -1626,7 +1614,8 @@ Extractors take the URL of a page to archive, write their output to the filesyst
 <br/><br/>
 <a href="https://twitter.com/ArchiveBoxApp"><img src="https://img.shields.io/badge/Tweet-%40ArchiveBoxApp-blue.svg?style=flat"/></a>&nbsp;
 <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?style=flat&label=Star+on+Github"/></a>&nbsp;
-<a href="https://zulip.archivebox.io/"><img src="https://img.shields.io/badge/Join_Our_Community-Zulip_Forum-%23B7EDFE.svg"/></a><br/><br/>
+<a href="https://zulip.archivebox.io/"><img src="https://img.shields.io/badge/Join_Our_Community-Zulip_Forum-%23B7EDFE.svg"/></a><br/>
+<sup>ArchiveBox was started by <a href="https://docs.sweeting.me/s/blog#About">Nick Sweeting</a> in 2017, and has <a href="https://star-history.com/#archivebox/archivebox&Date">grown steadily</a> with help from our <a href="https://github.com/ArchiveBox/ArchiveBox/graphs/contributors">amazing contributors</a>.</sup>
 <hr/>
 <i>✨ Have spare CPU/disk/bandwidth after all your 网站存档爬 and want to help the world?<br/>Check out our <a href="https://github.com/ArchiveBox/good-karma-kit">Good Karma Kit</a>...</i>
 </div>
@@ -3,4 +3,4 @@ ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E
 select = F,E9,W
 max-line-length = 130
 max-complexity = 10
-exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv,data,data*
+exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv
@@ -1,172 +0,0 @@
-# ArchiveBox UI
-
-## Page: Getting Started
-
-### What do you want to capture?
-
- Save some URLs now -> [Add page]
-    - Paste some URLs to archive now
-    - Upload a file containing URLs (bookmarks.html export, RSS.xml feed, markdown file, word doc, PDF, etc.)
-    - Pull in URLs to archive from a remote location (e.g. RSS feed URL, remote TXT file, JSON file, etc.)
-
- Import URLs from a browser -> [Import page]
-    - Desktop: Get the ArchiveBox Chrome/Firefox extension
-    - Mobile: Get the ArchiveBox iOS App / Android App
-    - Upload a bookmarks.html export file
-    - Upload a browser_history.sqlite3 export file
-
- Import URLs from a 3rd party bookmarking service -> [Sync page]
-    - Pocket
-    - Pinboard
-    - Instapaper
-    - Wallabag
-    - Zapier, N8N, IFTTT, etc.
-    - Upload a bookmarks.html export, bookmarks.json, RSS, etc. file
-
- Archive URLs on a schedule -> [Schedule page]
-
- Archive an entire website -> [Crawl page]
-    - What starting URL/domain?
-    - How deep?
-    - Follow links to external domains?
-    - Follow links to parent URLs?
-    - Maximum number of pages to save?
-    - Maximum number of requests/minute?
-
- Crawl for URLs with a search engine and save automatically
-    - 
- Some URLs on a schedule
- Save an entire website (e.g. `https://example.com`)
- Save results matching a search query (e.g. "site:example.com")
- Save a social media feed (e.g. `https://x.com/user/1234567890`)
-
--------------------------------------------------------------------------------
-
-### Crawls App
-
- Archive an entire website -> [Crawl page]
-    - What are the seed URLs?
-    - How many hops to follow?
-    - Follow links to external domains?
-    - Follow links to parent URLs?
-    - Maximum number of pages to save?
-    - Maximum number of requests/minute?
-
-
--------------------------------------------------------------------------------
-
-### Scheduler App
-
-
- Archive URLs on a schedule -> [Schedule page]
-    - What URL(s)?
-    - How often?
-    - Do you want to discard old snapshots after x amount of time?
-    - Any filter rules?
-    - Want to be notified when changes are detected -> redirect[Alerts app/create new alert(crawl=self)]
-
-
-* Choose Schedule check for new URLs: Schedule.objects.get(pk=xyz)
-    - 1 minute
-    - 5 minutes
-    - 1 hour
-    - 1 day
-
-    * Choose Destination Crawl to archive URLs using : Crawl.objects.get(pk=xyz)
-        - Tags
-        - Persona
-        - Created By ID
-        - Config
-        - Filters
-            - URL patterns to include
-            - URL patterns to exclude
-            - ONLY_NEW= Ignore URLs if already saved once / save URL each time it appears / only save is last save > x time ago
-
-
--------------------------------------------------------------------------------
-
-### Sources App (For managing sources that ArchiveBox pulls URLs in from)
-
- Add a new source to pull URLs in from (WIZARD)
-    - Choose URI:
-        - [x] Web UI
-        - [x] CLI
-        - Local filesystem path (directory to monitor for new files containing URLs)
-        - Remote URL (RSS/JSON/XML feed)
-        - Chrome browser profile sync (login using gmail to pull bookmarks/history)
-        - Pocket, Pinboard, Instapaper, Wallabag, etc.
-        - Zapier, N8N, IFTTT, etc.
-        - Local server filesystem path (directory to monitor for new files containing URLs)
-        - Google drive (directory to monitor for new files containing URLs)
-        - Remote server FTP/SFTP/SCP path (directory to monitor for new files containing URLs)
-        - AWS/S3/B2/GCP bucket (directory to monitor for new files containing URLs)
-        - XBrowserSync (login to pull bookmarks)
-    - Choose extractor
-        - auto
-        - RSS
-        - Pocket
-        - etc.
-    - Specify extra Config, e.g.
-        - credentials
-        - extractor tuning options (e.g. verify_ssl, cookies, etc.)
-
- Provide credentials for the source
-    - API Key
-    - Username / Password
-    - OAuth
-
--------------------------------------------------------------------------------
-
-### Alerts App
-
- Create a new alert, choose condition
-    - Get notified when a site goes down (<x% success ratio for Snapshots)
-    - Get notified when a site changes visually more than x% (screenshot diff)
-    - Get notified when a site's text content changes more than x% (text diff)
-    - Get notified when a keyword appears
-    - Get notified when a keyword dissapears
-    - When an AI prompt returns some result
-
- Choose alert threshold:
-    - any condition is met
-    - all conditions are met
-    - condition is met for x% of URLs
-    - condition is met for x% of time
-
- Choose how to notify: (List[AlertDestination])
-    - maximum alert frequency
-    - destination type: email / Slack / Webhook / Google Sheet / logfile
-    - destination info:
-        - email address(es)
-        - Slack channel
-        - Webhook URL
-
- Choose scope:
-    - Choose ArchiveResult scope (extractors): (a query that returns ArchiveResult.objects QuerySet)
-        - All extractors
-        - Only screenshots
-        - Only readability / mercury text
-        - Only video
-        - Only html
-        - Only headers
-
-    - Choose Snapshot scope (URL): (a query that returns Snapshot.objects QuerySet)
-        - All domains
-        - Specific domain
-        - All domains in a tag
-        - All domains in a tag category
-        - All URLs matching a certain regex pattern
-
-    - Choose crawl scope: (a query that returns Crawl.objects QuerySet)
-        - All crawls
-        - Specific crawls
-        - crawls by a certain user
-        - crawls using a certain persona
-
-
-class AlertDestination(models.Model):
-    destination_type: [email, slack, webhook, google_sheet, local logfile, b2/s3/gcp bucket, etc.]
-    maximum_frequency
-    filter_rules
-    credentials
-    alert_template: JINJA2 json/text template that gets populated with alert contents
@@ -0,0 +1 @@
+../LICENSE
@@ -1,149 +1,4 @@
-#!/usr/bin/env python3
-
-# Welcome to the ArchiveBox source code! Thanks for checking it out!
-#
-# "We are swimming upstream against a great torrent of disorganization.
-# In this, our main obligation is to establish arbitrary enclaves of order and system.
-# It is the greatest possible victory to be, to continue to be, and to have been.
-# No defeat can deprive us of the success of having existed for some moment of time
-# in a universe that seems indifferent to us."
-# --Norber Weiner
-
 __package__ = 'archivebox'

-import os
-import sys
-from pathlib import Path
-from typing import cast

-ASCII_LOGO = """
- █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
-██╔══██╗██╔══██╗██╔════╝██║  ██║██║██║   ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
-███████║██████╔╝██║     ███████║██║██║   ██║█████╗   ██████╔╝██║   ██║ ╚███╔╝ 
-██╔══██║██╔══██╗██║     ██╔══██║██║╚██╗ ██╔╝██╔══╝   ██╔══██╗██║   ██║ ██╔██╗ 
-██║  ██║██║  ██║╚██████╗██║  ██║██║ ╚████╔╝ ███████╗ ██████╔╝╚██████╔╝██╔╝ ██╗
-╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
-"""
-
-# make sure PACKAGE_DIR is in sys.path so we can import all subfolders
-# without necessarily waiting for django to load them thorugh INSTALLED_APPS
-PACKAGE_DIR = Path(__file__).resolve().parent
-if str(PACKAGE_DIR) not in sys.path:
-    sys.path.append(str(PACKAGE_DIR))
-os.environ['DJANGO_SETTINGS_MODULE'] = 'core.settings'
-os.environ['TZ'] = 'UTC'
-
-# detect ArchiveBox user's UID/GID based on data dir ownership
-from .config.permissions import drop_privileges                 # noqa
-drop_privileges()
-
-from .misc.checks import check_not_root, check_io_encoding      # noqa
-check_not_root()
-check_io_encoding()
-
-# print('INSTALLING MONKEY PATCHES')
-from .monkey_patches import *                    # noqa
-# print('DONE INSTALLING MONKEY PATCHES')
-
-
-# print('LOADING VENDORED LIBRARIES')
-from .pkgs import load_vendored_pkgs             # noqa
-load_vendored_pkgs()
-# print('DONE LOADING VENDORED LIBRARIES')
-
-# Load ABX Plugin Specifications + Default Implementations
-import abx                                       # noqa
-import abx_spec_archivebox                       # noqa
-import abx_spec_config                           # noqa
-import abx_spec_pydantic_pkgr                    # noqa
-import abx_spec_django                           # noqa
-import abx_spec_searchbackend                    # noqa
-
-abx.pm.add_hookspecs(abx_spec_config.PLUGIN_SPEC)
-abx.pm.register(abx_spec_config.PLUGIN_SPEC())
-
-abx.pm.add_hookspecs(abx_spec_pydantic_pkgr.PLUGIN_SPEC)
-abx.pm.register(abx_spec_pydantic_pkgr.PLUGIN_SPEC())
-
-abx.pm.add_hookspecs(abx_spec_django.PLUGIN_SPEC)
-abx.pm.register(abx_spec_django.PLUGIN_SPEC())
-
-abx.pm.add_hookspecs(abx_spec_searchbackend.PLUGIN_SPEC)
-abx.pm.register(abx_spec_searchbackend.PLUGIN_SPEC())
-
-# Cast to ArchiveBoxPluginSpec to enable static type checking of pm.hook.call() methods
-abx.pm = cast(abx.ABXPluginManager[abx_spec_archivebox.ArchiveBoxPluginSpec], abx.pm)
-pm = abx.pm
-
-
-# Load all pip-installed ABX-compatible plugins
-ABX_ECOSYSTEM_PLUGINS = abx.get_pip_installed_plugins(group='abx')
-
-# Load all built-in ArchiveBox plugins
-ARCHIVEBOX_BUILTIN_PLUGINS = {
-    'config': PACKAGE_DIR / 'config',
-    'core': PACKAGE_DIR / 'core',
-    # 'search': PACKAGE_DIR / 'search',
-    # 'core': PACKAGE_DIR / 'core',
-}
-
-# Load all user-defined ArchiveBox plugins
-USER_PLUGINS = abx.find_plugins_in_dir(Path(os.getcwd()) / 'user_plugins')
-
-# Import all plugins and register them with ABX Plugin Manager
-ALL_PLUGINS = {**ABX_ECOSYSTEM_PLUGINS, **ARCHIVEBOX_BUILTIN_PLUGINS, **USER_PLUGINS}
-LOADED_PLUGINS = abx.load_plugins(ALL_PLUGINS)
-
-# Setup basic config, constants, paths, and version
-from .config.constants import CONSTANTS                         # noqa
-from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR    # noqa
-from .config.version import VERSION                             # noqa
-
-__version__ = VERSION
-__author__ = 'ArchiveBox'
-__license__ = 'MIT'
-
-ASCII_ICON = """
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-         ██                                                                            ██          
-         ██                                                                            ██        
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                    ████████████████████████████████████                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ████████████████████████████████████                    ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                 ██████████████████████████████████████████                 ██          
-         ██                 ██████████████████████████████████████████                 ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██        
-         ████████████████████████████████████████████████████████████████████████████████          
-"""
+from .monkey_patches import *
@@ -1,19 +1,11 @@
 #!/usr/bin/env python3
-"""This is the main entry point for the ArchiveBox CLI."""
+
 __package__ = 'archivebox'

-import archivebox      # noqa # make sure monkey patches are applied before anything else
 import sys

 from .cli import main

-ASCII_LOGO_MINI = r"""
-     _             _     _           ____            
-    / \   _ __ ___| |__ (_)_   _____| __ )  _____  __
-   / _ \ | '__/ __| '_ \| \ \ / / _ \  _ \ / _ \ \/ /
-  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  < 
- /_/   \_\_|  \___|_| |_|_| \_/ \___|____/ \___/_/\_\
-"""

 if __name__ == '__main__':
    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1,6 +1,4 @@
-__package__ = 'archivebox.abid_utils'
-
-from typing import NamedTuple, Any, Union, Dict
+from typing import NamedTuple, Any, Union, Optional

 import ulid
 import uuid6
@@ -11,7 +9,6 @@ from uuid import UUID
 from typeid import TypeID            # type: ignore[import-untyped]
 from datetime import datetime

-from archivebox.misc.util import enforce_types


 ABID_PREFIX_LEN = 4
@@ -111,10 +108,9 @@ class ABID(NamedTuple):
 ####################################################


-@enforce_types
 def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
    """
-    https://example.com -> 'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25' (example.com)
+    'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25'
    """
    if isinstance(uri, bytes):
        uri_str: str = uri.decode()
@@ -130,45 +126,35 @@ def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
        except AttributeError:
            pass
    
-    # the uri hash is the sha256 of the domain + salt
    uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')

    return hashlib.sha256(uri_bytes).hexdigest().upper()

-@enforce_types
-def abid_part_from_prefix(prefix: str) -> str:
+def abid_part_from_prefix(prefix: Optional[str]) -> str:
    """
    'snp_'
    """
-    # if prefix is None:
-    #     return 'obj_'
+    if prefix is None:
+        return 'obj_'

    prefix = prefix.strip('_').lower()
    assert len(prefix) == 3
    return prefix + '_'

-@enforce_types
-def abid_part_from_uri(uri: Any, salt: str=DEFAULT_ABID_URI_SALT) -> str:
+def abid_part_from_uri(uri: str, salt: str=DEFAULT_ABID_URI_SALT) -> str:
    """
    'E4A5CCD9'     # takes first 8 characters of sha256(url)
    """
-    uri = str(uri).strip()
-    assert uri not in ('None', '')
+    uri = str(uri)
    return uri_hash(uri, salt=salt)[:ABID_URI_LEN]

-@enforce_types
-def abid_part_from_ts(ts: datetime) -> str:
+def abid_part_from_ts(ts: Optional[datetime]) -> str:
    """
    '01HX9FPYTR'   # produces 10 character Timestamp section of ulid based on added date
    """
-    return str(ulid.from_timestamp(ts))[:ABID_TS_LEN]
+    return str(ulid.from_timestamp(ts) if ts else ulid.new())[:ABID_TS_LEN]

-@enforce_types
-def ts_from_abid(abid: str) -> datetime:
-    return ulid.parse(abid.split('_', 1)[-1]).timestamp().datetime
-
-@enforce_types
-def abid_part_from_subtype(subtype: str | int) -> str:
+def abid_part_from_subtype(subtype: str) -> str:
    """
    Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
    Also allows us to change the ulid spec later by putting special sigil values here.
@@ -179,7 +165,6 @@ def abid_part_from_subtype(subtype: str | int) -> str:

    return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()

-@enforce_types
 def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
    """
    'ZYEBQE'   # takes last 6 characters of randomness from existing legacy uuid db field
@@ -201,23 +186,17 @@ def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
    return str(rand)[-ABID_RAND_LEN:].upper()


-@enforce_types
-def abid_hashes_from_values(prefix: str, ts: datetime, uri: Any, subtype: str | int, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> Dict[str, str]:
-    return {
-        'prefix': abid_part_from_prefix(prefix),
-        'ts': abid_part_from_ts(ts),
-        'uri': abid_part_from_uri(uri, salt=salt),
-        'subtype': abid_part_from_subtype(subtype),
-        'rand': abid_part_from_rand(rand),
-        # 'salt': don't add this, salt combined with uri above to form a single hash
-    }
-
-@enforce_types
-def abid_from_values(prefix: str, ts: datetime, uri: str, subtype: str, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> ABID:
+def abid_from_values(prefix, ts, uri, subtype, rand, salt=DEFAULT_ABID_URI_SALT) -> ABID:
    """
    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
    """

-    abid = ABID(**abid_hashes_from_values(prefix, ts, uri, subtype, rand, salt=salt))
+    abid = ABID(
+        prefix=abid_part_from_prefix(prefix),
+        ts=abid_part_from_ts(ts),
+        uri=abid_part_from_uri(uri, salt=salt),
+        subtype=abid_part_from_subtype(subtype),
+        rand=abid_part_from_rand(rand),
+    )
    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
    return abid
@@ -1,174 +1,18 @@
-__package__ = 'archivebox.abid_utils'
-
-from typing import Any
-
-from django.contrib import admin, messages
-from django.core.exceptions import ValidationError
-from django.utils.html import format_html
-from django.utils.safestring import mark_safe
-from django.shortcuts import redirect
-
-from django_object_actions import DjangoObjectActions, action
-
-from archivebox.misc.util import parse_date
-
-from .abid import ABID
+from django.contrib import admin


-def highlight_diff(display_val: Any, compare_val: Any, invert: bool=False, color_same: str | None=None, color_diff: str | None=None):
-    """highlight each character in red that differs with the char at the same index in compare_val"""
-
-    display_val = str(display_val)
-    compare_val = str(compare_val)
-
-    if len(compare_val) < len(display_val):
-        compare_val += ' ' * (len(display_val) - len(compare_val))
-
-    similar_color, highlighted_color = color_same or 'inherit', color_diff or 'red'
-    if invert:
-        similar_color, highlighted_color = color_same or 'green', color_diff or 'inherit'
-
-    return mark_safe(''.join(
-        format_html('<span style="color: {};">{}</span>', highlighted_color, display_val[i])
-        if display_val[i] != compare_val[i] else
-        format_html('<span style="color: {};">{}</span>', similar_color, display_val[i])
-        for i in range(len(display_val))
-    ))
-
-def get_abid_info(self, obj, request=None):
-    from archivebox.api.auth import get_or_create_api_token
-    
-    try:
-        #abid_diff = f' != obj.ABID: {highlight_diff(obj.ABID, obj.abid)} ❌' if str(obj.ABID) != str(obj.abid) else ' == .ABID ✅'
-
-        fresh_values = obj.ABID_FRESH_VALUES
-        fresh_hashes = obj.ABID_FRESH_HASHES
-        fresh_diffs = obj.ABID_FRESH_DIFFS
-        fresh_abid = ABID(**fresh_hashes)
-        
-        fresh_abid_diff = f'❌ != &nbsp; .fresh_abid: {highlight_diff(fresh_abid, obj.ABID)}' if str(fresh_abid) != str(obj.ABID) else '✅'
-        fresh_uuid_diff = f'❌ != &nbsp; .fresh_uuid: {highlight_diff(fresh_abid.uuid, obj.ABID.uuid)}' if str(fresh_abid.uuid) != str(obj.ABID.uuid) else '✅'
-
-        id_pk_diff = f'❌ !=  .pk: {highlight_diff(obj.pk, obj.id)}' if str(obj.pk) != str(obj.id) else '✅'
-
-        fresh_ts = parse_date(fresh_values['ts']) or None
-        ts_diff = f'❌ != {highlight_diff( fresh_hashes["ts"], obj.ABID.ts)}' if  fresh_hashes["ts"] != obj.ABID.ts else '✅'
-
-        derived_uri = fresh_hashes['uri']
-        uri_diff = f'❌ != {highlight_diff(derived_uri, obj.ABID.uri)}' if derived_uri != obj.ABID.uri else '✅'
-
-        derived_subtype = fresh_hashes['subtype']
-        subtype_diff = f'❌ != {highlight_diff(derived_subtype, obj.ABID.subtype)}' if derived_subtype != obj.ABID.subtype else '✅'
-
-        derived_rand = fresh_hashes['rand']
-        rand_diff = f'❌ != {highlight_diff(derived_rand, obj.ABID.rand)}' if derived_rand != obj.ABID.rand else '✅'
-
-        return format_html(
-            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
-            '''
-            <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
-            <br/><hr/>
-            <div style="opacity: 0.8">
-            &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
-            &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; {}</small><br/>
-            &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</small><br/>
-            <hr/>
-            &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; {}</code> &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
-            &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; {}</code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px;"><b>{}</b></code> <span style="display:inline-block; vertical-align: -4px; width: 330px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}: <code style="user-select: all">{}</code></span><br/>
-            &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code><br/>
-            &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b>{}</b></code> {}: <code style="user-select: all">{}</code></code>
-            <br/><hr/>
-            <span style="color: #f375a0">{}</span> <code style="color: red"><b>{}</b></code> {}
-            </div>
-            ''',
-            obj.api_url + (f'?api_key={get_or_create_api_token(request.user)}' if request and request.user else ''), obj.api_url, obj.api_docs_url,
-            highlight_diff(obj.id, obj.ABID.uuid, invert=True), mark_safe(id_pk_diff),
-            highlight_diff(obj.ABID.uuid, obj.id, invert=True), mark_safe(fresh_uuid_diff),
-            highlight_diff(obj.abid, fresh_abid), mark_safe(fresh_abid_diff),
-            # str(fresh_abid.uuid), mark_safe(fresh_uuid_diff),
-            # str(fresh_abid), mark_safe(fresh_abid_diff),
-            highlight_diff(obj.ABID.ts,  fresh_hashes['ts']), highlight_diff(str(obj.ABID.uuid)[0:14], str(fresh_abid.uuid)[0:14]), mark_safe(ts_diff), obj.abid_ts_src, fresh_ts and fresh_ts.isoformat(),
-            highlight_diff(obj.ABID.uri, derived_uri), highlight_diff(str(obj.ABID.uuid)[14:26], str(fresh_abid.uuid)[14:26]), mark_safe(uri_diff), obj.abid_uri_src, str(fresh_values['uri']),
-            highlight_diff(obj.ABID.subtype, derived_subtype), highlight_diff(str(obj.ABID.uuid)[26:28], str(fresh_abid.uuid)[26:28]), mark_safe(subtype_diff), obj.abid_subtype_src, str(fresh_values['subtype']),
-            highlight_diff(obj.ABID.rand, derived_rand), highlight_diff(str(obj.ABID.uuid)[28:36], str(fresh_abid.uuid)[28:36]), mark_safe(rand_diff), obj.abid_rand_src, str(fresh_values['rand'])[-7:],
-            'Some values the ABID depends on have changed since the ABID was issued:' if fresh_diffs else '',
-            ", ".join(diff['abid_src'] for diff in fresh_diffs.values()),
-            '(clicking "Regenerate ABID" in the upper right will assign a new ABID, breaking any external references to the old ABID)' if fresh_diffs else '',
-        )
-    except Exception as e:
-        # import ipdb; ipdb.set_trace()
-        return str(e)
-
-
-class ABIDModelAdmin(DjangoObjectActions, admin.ModelAdmin):
-    list_display = ('created_at', 'created_by', 'abid')
-    sort_fields = ('created_at', 'created_by', 'abid')
-    readonly_fields = ('created_at', 'modified_at', 'abid_info')
-    # fields = [*readonly_fields]
-    
-    change_actions = ("regenerate_abid",)
-    # changelist_actions = ("regenerate_abid",)
-
-    def _get_obj_does_not_exist_redirect(self, request, opts, object_id):
-        try:
-            object_pk = self.model.id_from_abid(object_id)
-            return redirect(self.request.path.replace(object_id, object_pk), permanent=False)
-        except (self.model.DoesNotExist, ValidationError):
-            pass
-        return super()._get_obj_does_not_exist_redirect(request, opts, object_id)       # type: ignore
-    
-    def queryset(self, request):
-        self.request = request
-        return super().queryset(request)                                                # type: ignore
-    
-    def change_view(self, request, object_id, form_url="", extra_context=None):
-        self.request = request
-        return super().change_view(request, object_id, form_url, extra_context)
+class ABIDModelAdmin(admin.ModelAdmin):
+    list_display = ('created', 'created_by', 'abid', '__str__')
+    sort_fields = ('created', 'created_by', 'abid', '__str__')
+    readonly_fields = ('abid', 'created', '__str__')

    def get_form(self, request, obj=None, **kwargs):
-        self.request = request
        form = super().get_form(request, obj, **kwargs)
        if 'created_by' in form.base_fields:
            form.base_fields['created_by'].initial = request.user
-            
-        if obj:
-            if obj.ABID_FRESH_DIFFS:
-                messages.warning(request, "The ABID is not in sync with the object! See the API Identifiers section below for more info...")
-
        return form

-    def get_formset(self, request, formset=None, obj=None, **kwargs):
-        formset = super().get_formset(request, formset, obj, **kwargs)                  # type: ignore
-        formset.form.base_fields['created_at'].disabled = True
-        
-        return formset
-
-    def save_model(self, request, obj, form, change):
-        self.request = request
-
-        old_abid = getattr(obj, '_previous_abid', None) or obj.abid
-
-        super().save_model(request, obj, form, change)
-        obj.refresh_from_db()
-
-        new_abid = obj.abid
-        if new_abid != old_abid:
-            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
-        # import ipdb; ipdb.set_trace()
-
-    @admin.display(description='API Identifiers')
-    def abid_info(self, obj):
-        return get_abid_info(self, obj, request=self.request)
-
-    @action(label="Regenerate ABID", description="Re-Generate the ABID based on fresh values")
-    def regenerate_abid(self, request, obj):
-        old_abid = str(obj.abid)
-        obj.abid = obj.issue_new_abid(overwrite=True)
-        obj.save()
-        obj.refresh_from_db()
-        new_abid = str(obj.abid)
-
-        if new_abid != old_abid:
-            messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
-        else:
-            messages.success(request, "The ABID was not regenerated, it is already up-to-date with the object.")
+    # def save_model(self, request, obj, form, change):
+    #     if getattr(obj, 'created_by_id', None) in (None, get_or_create_system_user_pk()):
+    #         obj.created_by = request.user
+    #     obj.save()
@@ -1,21 +1,23 @@
 """
 This file provides the Django ABIDField and ABIDModel base model to inherit from.
+
+It implements the ArchiveBox ID (ABID) interfaces including abid_values, generate_abid, .abid, .uuid, .id.
 """

+from typing import Any, Dict, Union, List, Set, NamedTuple, cast

-from typing import Any, Dict, Union, List, Set, cast
-
-from uuid import uuid4
+from ulid import ULID
+from uuid import uuid4, UUID
+from typeid import TypeID            # type: ignore[import-untyped]
+from datetime import datetime
 from functools import partial
 from charidfield import CharIDField  # type: ignore[import-untyped]

-from django.contrib import admin
-from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
+from django.conf import settings
 from django.db import models
 from django.utils import timezone
 from django.db.utils import OperationalError
 from django.contrib.auth import get_user_model
-from django.urls import reverse_lazy

 from django_stubs_ext.db.models import TypedModelMeta

@@ -27,9 +29,7 @@ from .abid import (
    DEFAULT_ABID_PREFIX,
    DEFAULT_ABID_URI_SALT,
    abid_part_from_prefix,
-    abid_hashes_from_values,
-    ts_from_abid,
-    abid_part_from_ts,
+    abid_from_values
 )

 ####################################################
@@ -57,306 +57,142 @@ def get_or_create_system_user_pk(username='system'):
        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]

    # otherwise, create a dedicated "system" user
-    user, _was_created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
+    user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
    return user.pk


-class AutoDateTimeField(models.DateTimeField):
-    # def pre_save(self, model_instance, add):
-    #     return timezone.now()
-    pass
-
-class ABIDError(Exception):
-    pass
-
-
 class ABIDModel(models.Model):
    """
    Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
    """
-    abid_prefix: str = DEFAULT_ABID_PREFIX            # e.g. 'tag_'
-    abid_ts_src = 'self.created_at'                  # e.g. 'self.created_at'
-    abid_uri_src = 'None'                            # e.g. 'self.uri'                (MUST BE SET)
-    abid_subtype_src = 'self.__class__.__name__'     # e.g. 'self.extractor'
-    abid_rand_src = 'self.id'                        # e.g. 'self.uuid' or 'self.id'
-    abid_salt: str = DEFAULT_ABID_URI_SALT           # combined with self.uri to anonymize hashes on a per-install basis (default is shared globally with all users, means everyone will hash ABC to -> 123 the same around the world, makes it easy to share ABIDs across installs and see if they are for the same URI. Change this if you dont want your hashes to be guessable / in the same hash space as all other users)
-    abid_drift_allowed: bool = False                 # set to True to allow abid_field values to change after a fixed ABID has been issued (NOT RECOMMENDED: means values can drift out of sync from original ABID)
+    abid_prefix: str = DEFAULT_ABID_PREFIX  # e.g. 'tag_'
+    abid_ts_src = 'None'                    # e.g. 'self.created'
+    abid_uri_src = 'None'                   # e.g. 'self.uri'
+    abid_subtype_src = 'None'               # e.g. 'self.extractor'
+    abid_rand_src = 'None'                  # e.g. 'self.uuid' or 'self.id'

-    # id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
-    # abid = ABIDField(prefix=abid_prefix)
+    # id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
+    # uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
+    abid = ABIDField(prefix=abid_prefix)

-    # created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    # created_at = AutoDateTimeField(default=None, null=False, db_index=True)
-    # modified_at = models.DateTimeField(auto_now=True)
-
-    _prefetched_objects_cache: Dict[str, Any]
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)

    class Meta(TypedModelMeta):
        abstract = True

-    @admin.display(description='Summary')
-    def __str__(self) -> str:
-        return f'[{self.abid or (self.abid_prefix + "NEW")}] {self.__class__.__name__} {eval(self.abid_uri_src)}'
+    def save(self, *args: Any, **kwargs: Any) -> None:
+        # when first creating a row, self.ABID is the source of truth
+        # overwrite default prefilled self.id & self.abid with generated self.ABID value
+        if self._state.adding or not self.id:
+            self.id = self.ABID.uuid
+        if self._state.adding or not self.abid:
+            self.abid = str(self.ABID)

-    def __init__(self, *args: Any, **kwargs: Any) -> None:
-        """Overriden __init__ method ensures we have a stable creation timestamp that fields can use within initialization code pre-saving to DB."""
-        super().__init__(*args, **kwargs)
-        # pre-compute a stable timestamp of the obj init time (with abid.ts precision limit applied) for use when object is first created,
-        # some other fields depend on a timestamp at creation time, and it's nice to have one common timestamp they can all share.
-        # Used as an alternative to auto_now_add=True + auto_now=True which can produce two different times & requires saving to DB to get the TS.
-        # (ordinarily fields cant depend on other fields until the obj is saved to db and recalled)
-        self._init_timestamp = ts_from_abid(abid_part_from_ts(timezone.now()))
-
-    def clean(self, abid_drift_allowed: bool | None=None) -> None:
-        if self._state.adding:
-            # only runs once when a new object is first saved to the DB
-            # sets self.id, self.pk, self.created_by, self.created_at, self.modified_at
-            self._previous_abid = None
-            self.abid = str(self.issue_new_abid())
-
-        else:
-            # otherwise if updating, make sure none of the field changes would invalidate existing ABID
-            abid_diffs = self.ABID_FRESH_DIFFS
-            if abid_diffs:
-                # change has invalidated the existing ABID, raise a nice ValidationError pointing out which fields caused the issue
-
-                keys_changed = ', '.join(diff['abid_src'] for diff in abid_diffs.values())
-                full_summary = (
-                    f"This {self.__class__.__name__}(abid={str(self.ABID)}) was assigned a fixed, unique ID (ABID) based on its contents when it was created. " +
-                    f"\nYou must reduce your changes to not affect these fields [{keys_changed}], or create a new {self.__class__.__name__} object instead."
-                )
-
-                change_error = ValidationError({
-                    **{
-                        # url: ValidationError('Cannot update self.url= https://example.com/old -> https://example.com/new ...')
-                        diff['abid_src'].replace('self.', '')
-                            if (diff['old_val'] != diff['new_val']) and hasattr(self, diff['abid_src'].replace('self.', ''))
-                            else NON_FIELD_ERRORS
-                        : ValidationError(
-                            'Cannot update %(abid_src)s= "%(old_val)s" -> "%(new_val)s" (would alter %(model)s.ABID.%(key)s=%(old_hash)s to %(new_hash)s)',
-                            code='ABIDConflict',
-                            params=diff,
-                        )
-                        for diff in abid_diffs.values()
-                    },
-                    NON_FIELD_ERRORS: ValidationError(full_summary),
-                })
-
-                allowed_to_invalidate_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed
-                if allowed_to_invalidate_abid:
-                    # print(f'\n#### WARNING: Change allowed despite it invalidating the ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})!', self.abid)
-                    # print(change_error)
-                    # print('--------------------------------------------------------------------------------------------------')
-                    pass
-                else:
-                    print(f'\n#### ERROR:   Change blocked because it would invalidate ABID of an existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})', self.abid)
-                    print(change_error)
-                    print('--------------------------------------------------------------------------------------------------')
-                    raise change_error
-
-    def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None:
-        """Overriden save method ensures new ABID is generated while a new object is first saving."""
-
-        self.clean(abid_drift_allowed=abid_drift_allowed)
-
-        return super().save(*args, **kwargs)
-    
-    @classmethod
-    def id_from_abid(cls, abid: str) -> str:
-        return str(cls.objects.only('pk').get(abid=cls.abid_prefix + str(abid).split('_', 1)[-1]).pk)
+        super().save(*args, **kwargs)
+        assert str(self.id) == str(self.ABID.uuid), f'self.id {self.id} does not match self.ABID {self.ABID.uuid}'
+        assert str(self.abid) == str(self.ABID), f'self.abid {self.id} does not match self.ABID {self.ABID.uuid}'

    @property
-    def ABID_SOURCES(self) -> Dict[str, str]:
-        """"Get the dict of fresh ABID component values based on the live object's properties."""
-        assert self.abid_prefix
+    def abid_values(self) -> Dict[str, Any]:
        return {
-            'prefix': 'self.abid_prefix',             # defined as static class vars at build time
-            'ts': self.abid_ts_src,
-            'uri': self.abid_uri_src,
-            'subtype': self.abid_subtype_src,
-            'rand': self.abid_rand_src,
-            'salt': 'self.abid_salt',                 # defined as static class vars at build time
+            'prefix': self.abid_prefix,
+            'ts': eval(self.abid_ts_src),
+            'uri': eval(self.abid_uri_src),
+            'subtype': eval(self.abid_subtype_src),
+            'rand': eval(self.abid_rand_src),
        }

-    @property
-    def ABID_FRESH_VALUES(self) -> Dict[str, Any]:
-        """"Get the dict of fresh ABID component values based on the live object's properties."""
-        abid_sources = self.ABID_SOURCES
-        assert all(src != 'None' for src in abid_sources.values())
-        return {
-            'prefix': eval(abid_sources['prefix']),
-            'ts': eval(abid_sources['ts']),
-            'uri': eval(abid_sources['uri']),
-            'subtype': eval(abid_sources['subtype']),
-            'rand': eval(abid_sources['rand']),
-            'salt': eval(abid_sources['salt']),
-        }
-    
-    @property
-    def ABID_FRESH_HASHES(self) -> Dict[str, str]:
-        """"Get the dict of fresh ABID component hashes based on the live object's properties."""
-        abid_values = self.ABID_FRESH_VALUES
-        assert all(val for val in abid_values.values())
-        return abid_hashes_from_values(
-            prefix=abid_values['prefix'],
-            ts=abid_values['ts'],
-            uri=abid_values['uri'],
-            subtype=abid_values['subtype'],
-            rand=abid_values['rand'],
-            salt=abid_values['salt'],
+    def generate_abid(self) -> ABID:
+        """
+        Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+        """
+        prefix, ts, uri, subtype, rand = self.abid_values.values()
+
+        if (not prefix) or prefix == DEFAULT_ABID_PREFIX:
+            suggested_abid = self.__class__.__name__[:3].lower()
+            raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
+
+        if not ts:
+            # default to unix epoch with 00:00:00 UTC
+            ts = datetime.fromtimestamp(0, timezone.utc)     # equivalent to: ts = datetime.utcfromtimestamp(0)
+            print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
+
+        if not uri:
+            uri = str(self)
+            print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
+
+        if not subtype:
+            subtype = self.__class__.__name__
+            print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
+
+        if not rand:
+            rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
+            print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
+
+        abid = abid_from_values(
+            prefix=prefix,
+            ts=ts,
+            uri=uri,
+            subtype=subtype,
+            rand=rand,
+            salt=DEFAULT_ABID_URI_SALT,
        )
-    
-    @property
-    def ABID_FRESH_DIFFS(self) -> Dict[str, Dict[str, Any]]:
-        """Get the dict of discrepancies between the existing saved ABID and a new fresh ABID computed based on the live object."""
-        existing_abid = self.ABID
-        existing_values = {} if self._state.adding else self.__class__.objects.get(pk=self.pk).ABID_FRESH_VALUES
-        abid_sources = self.ABID_SOURCES
-        fresh_values = self.ABID_FRESH_VALUES
-        fresh_hashes = self.ABID_FRESH_HASHES
-        return {
-            key: {
-                'key': key,
-                'model': self.__class__.__name__,
-                'pk': self.pk,
-                'abid_src': abid_sources[key],
-                'old_val': existing_values.get(key, None),
-                'old_hash': getattr(existing_abid, key),
-                'new_val': fresh_values[key],
-                'new_hash': new_hash,
-                'summary': f'{abid_sources[key]}= "{existing_values.get(key, None)}" -> "{fresh_values[key]}" (would alter {self.__class__.__name__.lower()}.ABID.{key}={getattr(existing_abid, key)} to {new_hash})',
-            }
-            for key, new_hash in fresh_hashes.items()
-            if getattr(existing_abid, key) != new_hash
-        }
-
-    def issue_new_abid(self, overwrite=False) -> ABID:
-        """
-        Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB).
-        """
-        if not overwrite:
-            assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
-        assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'
-
-        # Setup Field defaults to be ready for ABID generation
-        self.abid = None
-        self.id = self.id or uuid4()
-        self.pk = self.id
-        self.created_at = self.created_at or self._init_timestamp  # cut off precision to match precision of TS component
-        self.modified_at = self.modified_at or self.created_at
-        self.created_by_id = (hasattr(self, 'created_by_id') and self.created_by_id) or get_or_create_system_user_pk()
-        
-        # Compute fresh ABID values & hashes based on object's live properties
-        abid_fresh_values = self.ABID_FRESH_VALUES
-        assert all(abid_fresh_values.values()), f'All ABID_FRESH_VALUES must be set {abid_fresh_values}'
-        abid_fresh_hashes = self.ABID_FRESH_HASHES
-        assert all(abid_fresh_hashes.values()), f'All ABID_FRESH_HASHES must be able to be generated {abid_fresh_hashes}'
-        
-        new_abid = ABID(**abid_fresh_hashes)
-        
-        assert new_abid.ulid and new_abid.uuid and new_abid.typeid, f'Failed to calculate {abid_fresh_values["prefix"]}_ABID for {self.__class__.__name__}'
-
-        return new_abid
+        assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
+        return abid

    @property
    def ABID(self) -> ABID:
        """
-        Get the object's existing ABID (from self.abid if it's already saved to DB, otherwise generated fresh)
-        e.g. -> ABID(ts='01HX9FPYTR', uri='E4A5CCD9', subtype='00', rand='ZYEBQE')
+        ULIDParts(timestamp='01HX9FPYTR', url='E4A5CCD9', subtype='00', randomness='ZYEBQE')
        """
-
-        if self.abid:
-            return ABID.parse(cast(str, self.abid))
        
-        return self.issue_new_abid()
-
-    # These are all example helpers to make it easy to access alternate formats of the ABID.*, only add them if you actually need them
-    # @property
-    # def UUID(self) -> UUID:
-    #     """
-    #     Get a uuid.UUID (v4) representation of the object's ABID.
-    #     """
-    #     return self.ABID.uuid
-    
-    # @property
-    # def uuid(self) -> str:
-    #     """
-    #     Get a str uuid.UUID (v4) representation of the object's ABID.
-    #     """
-    #     return str(self.ABID.uuid)
-    
-    # @property
-    # def ULID(self) -> ULID:
-    #     """
-    #     Get a ulid.ULID representation of the object's ABID.
-    #     """
-    #     return self.ABID.ulid
-
-    # @property
-    # def TypeID(self) -> TypeID:
-    #     """
-    #     Get a typeid.TypeID (stripe-style) representation of the object's ABID.
-    #     """
-    #     return self.ABID.typeid
-    
-    @property
-    def api_url(self) -> str:
-        """
-        Compute the REST API URL to access this object.
-        e.g. /api/v1/core/snapshot/snp_01BJQMF54D093DXEAWZ6JYRP
-        """
-        return reverse_lazy('api-1:get_any', args=[self.abid])  # + f'?api_key={get_or_create_api_token(request.user)}'
-
-    @property
-    def api_docs_url(self) -> str:
-        """
-        Compute the REST API Documentation URL to learn about accessing this object.
-        e.g. /api/v1/docs#/Core%20Models/api_v1_core_get_snapshots
-        """
-        return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}'
-
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
-    def get_absolute_url(self):
-        return self.api_docs_url
-
-
-
-class ModelWithHealthStats(models.Model):
-    num_uses_failed = models.PositiveIntegerField(default=0)
-    num_uses_succeeded = models.PositiveIntegerField(default=0)
-    
-    class Meta:
-        abstract = True
-    
-    def record_health_failure(self) -> None:
-        self.num_uses_failed += 1
-        self.save()
-
-    def record_health_success(self) -> None:
-        self.num_uses_succeeded += 1
-        self.save()
-        
-    def reset_health(self) -> None:
-        # move all the failures to successes when resetting so we dont lose track of the total count
-        self.num_uses_succeeded = self.num_uses_failed + self.num_uses_succeeded
-        self.num_uses_failed = 0
-        self.save()
+        # if object is not yet saved to DB, always generate fresh ABID from values
+        if self._state.adding:
+            return self.generate_abid()
        
+        # otherwise DB is single source of truth, load ABID from existing db pk
+        abid: ABID | None = None
+        try:
+            abid = abid or ABID.parse(self.pk)
+        except Exception:
+            pass
+
+        try:
+            abid = abid or ABID.parse(self.id)
+        except Exception:
+            pass
+
+        try:
+            abid = abid or ABID.parse(cast(str, self.abid))
+        except Exception:
+            pass
+
+        abid = abid or self.generate_abid()
+
+        return abid
+
    @property
-    def health(self) -> int:
-        total_uses = max((self.num_uses_failed + self.num_uses_succeeded, 1))
-        success_pct = (self.num_uses_succeeded / total_uses) * 100
-        return round(success_pct)
-
-
-
-
-
+    def ULID(self) -> ULID:
+        """
+        Get a ulid.ULID representation of the object's ABID.
+        """
+        return self.ABID.ulid

+    @property
+    def UUID(self) -> UUID:
+        """
+        Get a uuid.UUID (v4) representation of the object's ABID.
+        """
+        return self.ABID.uuid

+    @property
+    def TypeID(self) -> TypeID:
+        """
+        Get a typeid.TypeID (stripe-style) representation of the object's ABID.
+        """
+        return self.ABID.typeid



@@ -419,7 +255,6 @@ def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDMode
    """
    Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
    e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
-    Honestly should only be used for debugging, no reason to expose this ability to users.
    """

    # convert str to ABID if necessary
@@ -469,7 +304,7 @@ def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDMode
                )

            for obj in qs:
-                if abid in (str(obj.ABID), str(obj.id), str(obj.pk), str(obj.abid)):
+                if obj.generate_abid() == abid:
                    # found exact match, no need to keep iterating
                    return [obj]
                partial_matches.append(obj)
@@ -1,571 +0,0 @@
-__package__ = 'archivebox.actors'
-
-import os
-import time
-from typing import ClassVar, Generic, TypeVar, Any, Literal, Type, Iterable, cast, get_args
-from datetime import timedelta
-from multiprocessing import Process, cpu_count
-from threading import Thread, get_native_id
-
-import psutil
-from rich import print
-from statemachine import State, StateMachine, registry
-from statemachine.mixins import MachineMixin
-
-from django import db
-from django.db.models import QuerySet, sql, Q
-from django.db.models import Model as DjangoModel
-from django.utils import timezone
-from django.utils.functional import classproperty
-
-from .models import ModelWithStateMachine
-
-# from archivebox.logging_util import TimedProgress
-
-class ActorObjectAlreadyClaimed(Exception):
-    """Raised when the Actor tries to claim the next object from the queue but it's already been claimed by another Actor"""
-    pass
-
-class ActorQueueIsEmpty(Exception):
-    """Raised when the Actor tries to get the next object from the queue but it's empty"""
-    pass
-
-CPU_COUNT = cpu_count()
-DEFAULT_MAX_TICK_TIME = 60
-DEFAULT_MAX_CONCURRENT_ACTORS = min(max(2, int(CPU_COUNT * 0.6)), 8)   # 2 < 60% * num available cpu cores < 8
-
-limit = lambda n, max: min(n, max)
-
-LaunchKwargs = dict[str, Any]
-ObjectState = State | str
-ObjectStateList = Iterable[ObjectState]
-
-ModelType = TypeVar('ModelType', bound=ModelWithStateMachine)
-
-class ActorType(Generic[ModelType]):
-    """
-    Base class for all actors. Usage:
-    
-    class FaviconActor(ActorType[FaviconArchiveResult]):
-        FINAL_STATES: ClassVar[tuple[str, ...]] = ('succeeded', 'failed', 'skipped')
-        ACTIVE_STATE: ClassVar[str] = 'started'
-        
-        @classmethod
-        def qs(cls) -> QuerySet[FaviconArchiveResult]:
-            return ArchiveResult.objects.filter(extractor='favicon')   # or leave the default: FaviconArchiveResult.objects.all()
-    """
-    
-    ### Class attributes (defined on the class at compile-time when ActorType[MyModel] is defined)
-    Model: Type[ModelType]
-    StateMachineClass: Type[StateMachine]
-    
-    STATE_FIELD_NAME: ClassVar[str]
-    ACTIVE_STATE: ClassVar[ObjectState]
-    FINAL_STATES: ClassVar[ObjectStateList]
-    EVENT_NAME: ClassVar[str] = 'tick'                                    # the event name to trigger on the obj.sm: StateMachine (usually 'tick')
-    
-    CLAIM_ORDER: ClassVar[tuple[str, ...]] = ('retry_at',)                # the .order(*args) to claim the queue objects in, use ('?',) for random order
-    CLAIM_FROM_TOP_N: ClassVar[int] = CPU_COUNT * 10                      # the number of objects to consider when atomically getting the next object from the queue
-    CLAIM_ATOMIC: ClassVar[bool] = True                                   # whether to atomically fetch+claim the next object in one query, or fetch and lock it in two queries
-    
-    MAX_TICK_TIME: ClassVar[int] = DEFAULT_MAX_TICK_TIME                  # maximum duration in seconds to process a single object
-    MAX_CONCURRENT_ACTORS: ClassVar[int] = DEFAULT_MAX_CONCURRENT_ACTORS  # maximum number of concurrent actors that can be running at once
-    
-    _SPAWNED_ACTOR_PIDS: ClassVar[list[psutil.Process]] = []      # used to record all the pids of Actors spawned on the class
-    
-    ### Instance attributes (only used within an actor instance inside a spawned actor thread/process)
-    pid: int
-    idle_count: int = 0
-    launch_kwargs: LaunchKwargs = {}
-    mode: Literal['thread', 'process'] = 'process'
-    
-    def __init_subclass__(cls) -> None:
-        """
-        Executed at class definition time (i.e. during import of any file containing class MyActor(ActorType[MyModel]): ...).
-        Loads the django Model from the Generic[ModelType] TypeVar arg and populates any missing class-level config using it.
-        """
-        if getattr(cls, 'Model', None) is None:
-            cls.Model = cls._get_model_from_generic_typevar()
-        cls._populate_missing_classvars_from_model(cls.Model)
-    
-    def __init__(self, mode: Literal['thread', 'process']|None=None, **launch_kwargs: LaunchKwargs):
-        """
-        Executed right before the Actor is spawned to create a unique Actor instance for that thread/process.
-        actor_instance.runloop() is then executed from inside the newly spawned thread/process.
-        """
-        self.mode = mode or self.mode
-        self.launch_kwargs = launch_kwargs or dict(self.launch_kwargs)
-    
-
-    ### Private Helper Methods: Not desiged to be overridden by subclasses or called by anything outside of this class
-    
-    @classproperty
-    def name(cls) -> str:
-        return cls.__name__  # type: ignore
-    
-    def __str__(self) -> str:
-        return repr(self)
-    
-    def __repr__(self) -> str:
-        """-> FaviconActor[pid=1234]"""
-        label = 'pid' if self.mode == 'process' else 'tid'
-        return f'[underline]{self.name}[/underline]\\[{label}={self.pid}]'
-    
-    @staticmethod
-    def _state_to_str(state: ObjectState) -> str:
-        """Convert a statemachine.State, models.TextChoices.choices value, or Enum value to a str"""
-        return str(state.value) if isinstance(state, State) else str(state)
-    
-    @staticmethod
-    def _sql_for_select_top_n_candidates(qs: QuerySet, claim_from_top_n: int=CLAIM_FROM_TOP_N) -> tuple[str, tuple[Any, ...]]:
-        """Get the SQL for selecting the top N candidates from the queue (to claim one from)"""
-        queryset = qs.only('id')[:claim_from_top_n]
-        select_sql, select_params = compile_sql_select(queryset)
-        return select_sql, select_params
-    
-    @staticmethod
-    def _sql_for_update_claimed_obj(qs: QuerySet, update_kwargs: dict[str, Any]) -> tuple[str, tuple[Any, ...]]:
-        """Get the SQL for updating a claimed object to mark it as ACTIVE"""
-        # qs.update(status='started', retry_at=<now + MAX_TICK_TIME>)
-        update_sql, update_params = compile_sql_update(qs, update_kwargs=update_kwargs)
-        # e.g. UPDATE core_archiveresult SET status='%s', retry_at='%s' WHERE status NOT IN ('succeeded', 'failed', 'sealed', 'started') AND retry_at <= '2024-11-04 10:14:33.240903'
-        return update_sql, update_params
-    
-    @classmethod
-    def _get_model_from_generic_typevar(cls) -> Type[ModelType]:
-        """Get the django Model from the Generic[ModelType] TypeVar arg (and check that it inherits from django.db.models.Model)"""
-        # cls.__orig_bases__ is non-standard and may be removed in the future! if this breaks,
-        # we can just require the inerited class to define the Model as a classvar manually, e.g.:
-        #     class SnapshotActor(ActorType[Snapshot]):
-        #         Model: ClassVar[Type[Snapshot]] = Snapshot
-        # https://stackoverflow.com/questions/57706180/generict-base-class-how-to-get-type-of-t-from-within-instance
-        Model = get_args(cls.__orig_bases__[0])[0]   # type: ignore
-        assert issubclass(Model, DjangoModel), f'{cls.__name__}.Model must be a valid django Model'
-        return cast(Type[ModelType], Model)
-    
-    @staticmethod
-    def _get_state_machine_cls(Model: Type[ModelType]) -> Type[StateMachine]:
-        """Get the StateMachine class for the given django Model that inherits from MachineMixin"""
-        assert issubclass(Model, MachineMixin), f'{Model.__name__} must inherit from MachineMixin and define a .state_machine_name: str'
-        model_state_machine_name = getattr(Model, 'state_machine_name', None)
-        if model_state_machine_name:
-            StateMachine = registry.get_machine_cls(model_state_machine_name)
-            assert issubclass(StateMachine, StateMachine)
-            return StateMachine
-        raise NotImplementedError(f'ActorType[{Model.__name__}] must define .state_machine_name: str that points to a valid StateMachine')
-    
-    @classmethod
-    def _get_state_machine_instance(cls, obj: ModelType) -> StateMachine:
-        """Get the StateMachine instance for the given django Model instance (and check that it is a valid instance of cls.StateMachineClass)"""
-        obj_statemachine = None
-        state_machine_attr = getattr(obj, 'state_machine_attr', 'sm')
-        try:
-            obj_statemachine = getattr(obj, state_machine_attr)
-        except Exception:
-            pass
-        
-        if not isinstance(obj_statemachine, cls.StateMachineClass):
-            raise Exception(f'{cls.__name__}: Failed to find a valid StateMachine instance at {type(obj).__name__}.{state_machine_attr}')
-            
-        return obj_statemachine
-    
-    @classmethod
-    def _populate_missing_classvars_from_model(cls, Model: Type[ModelType]):
-        """Check that the class variables are set correctly based on the ModelType"""
-        
-        # check that Model is the same as the Generic[ModelType] parameter in the class definition
-        cls.Model = getattr(cls, 'Model', None) or Model
-        if cls.Model != Model:
-            raise ValueError(f'{cls.__name__}.Model must be set to the same Model as the Generic[ModelType] parameter in the class definition')
-        
-        # check that Model has a valid StateMachine with the required event defined on it
-        cls.StateMachineClass = getattr(cls, 'StateMachineClass', None) or cls._get_state_machine_cls(cls.Model)
-        assert isinstance(cls.EVENT_NAME, str), f'{cls.__name__}.EVENT_NAME must be a str, got: {type(cls.EVENT_NAME).__name__} instead'
-        assert hasattr(cls.StateMachineClass, cls.EVENT_NAME), f'StateMachine {cls.StateMachineClass.__name__} must define a {cls.EVENT_NAME} event ({cls.__name__}.EVENT_NAME = {cls.EVENT_NAME})'
-        
-        # check that Model uses .id as its primary key field
-        primary_key_field = cls.Model._meta.pk.name
-        if primary_key_field != 'id':
-            raise NotImplementedError(f'Actors currently only support models that use .id as their primary key field ({cls.__name__} uses {cls.__name__}.{primary_key_field} as primary key)')
-        
-        # check for STATE_FIELD_NAME classvar or set it from the model's state_field_name attr
-        if not getattr(cls, 'STATE_FIELD_NAME', None):
-            if hasattr(cls.Model, 'state_field_name'):
-                cls.STATE_FIELD_NAME = getattr(cls.Model, 'state_field_name')
-            else:
-                raise NotImplementedError(f'{cls.__name__} must define a STATE_FIELD_NAME: ClassVar[str] (e.g. "status") or have a .state_field_name attr on its Model')
-        assert isinstance(cls.STATE_FIELD_NAME, str), f'{cls.__name__}.STATE_FIELD_NAME must be a str, got: {type(cls.STATE_FIELD_NAME).__name__} instead'
-            
-        # check for FINAL_STATES classvar or set it from the model's final_states attr
-        if not getattr(cls, 'FINAL_STATES', None):
-            cls.FINAL_STATES = cls.StateMachineClass.final_states
-            if not cls.FINAL_STATES:
-                raise NotImplementedError(f'{cls.__name__} must define a non-empty FINAL_STATES: ClassVar[list[str]] (e.g. ["sealed"]) or have a {cls.Model.__name__}.state_machine_name pointing to a StateMachine that provides .final_states')
-        cls.FINAL_STATES = [cls._state_to_str(state) for state in cls.FINAL_STATES]
-        assert all(isinstance(state, str) for state in cls.FINAL_STATES), f'{cls.__name__}.FINAL_STATES must be a list[str], got: {type(cls.FINAL_STATES).__name__} instead'
-        
-        # check for ACTIVE_STATE classvar or set it from the model's active_state attr
-        if not getattr(cls, 'ACTIVE_STATE', None):
-            raise NotImplementedError(f'{cls.__name__} must define an ACTIVE_STATE: ClassVar[State] (e.g. SnapshotMachine.started) ({cls.Model.__name__}.{cls.STATE_FIELD_NAME} gets set to this value to mark objects as actively processing)')
-        assert isinstance(cls.ACTIVE_STATE, (State, str)), f'{cls.__name__}.ACTIVE_STATE must be a statemachine.State | str, got: {type(cls.ACTIVE_STATE).__name__} instead'
-        
-        # check the other ClassVar attributes for valid values
-        assert cls.CLAIM_ORDER and isinstance(cls.CLAIM_ORDER, tuple) and all(isinstance(order, str) for order in cls.CLAIM_ORDER), f'{cls.__name__}.CLAIM_ORDER must be a non-empty tuple[str, ...], got: {type(cls.CLAIM_ORDER).__name__} instead'
-        assert cls.CLAIM_FROM_TOP_N > 0, f'{cls.__name__}.CLAIM_FROM_TOP_N must be a positive int, got: {cls.CLAIM_FROM_TOP_N} instead'
-        assert cls.MAX_TICK_TIME >= 1, f'{cls.__name__}.MAX_TICK_TIME must be a positive int > 1, got: {cls.MAX_TICK_TIME} instead'
-        assert cls.MAX_CONCURRENT_ACTORS >= 1, f'{cls.__name__}.MAX_CONCURRENT_ACTORS must be a positive int >=1, got: {cls.MAX_CONCURRENT_ACTORS} instead'
-        assert isinstance(cls.CLAIM_ATOMIC, bool), f'{cls.__name__}.CLAIM_ATOMIC must be a bool, got: {cls.CLAIM_ATOMIC} instead'
-
-    @classmethod
-    def _fork_actor_as_thread(cls, **launch_kwargs: LaunchKwargs) -> int:
-        """Spawn a new background thread running the actor's runloop"""
-        actor = cls(mode='thread', **launch_kwargs)
-        bg_actor_thread = Thread(target=actor.runloop)
-        bg_actor_thread.start()
-        assert bg_actor_thread.native_id is not None
-        return bg_actor_thread.native_id
-    
-    @classmethod
-    def _fork_actor_as_process(cls, **launch_kwargs: LaunchKwargs) -> int:
-        """Spawn a new background process running the actor's runloop"""
-        actor = cls(mode='process', **launch_kwargs)
-        bg_actor_process = Process(target=actor.runloop)
-        bg_actor_process.start()
-        assert bg_actor_process.pid is not None
-        cls._SPAWNED_ACTOR_PIDS.append(psutil.Process(pid=bg_actor_process.pid))
-        return bg_actor_process.pid
-    
-    
-    ### Class Methods: Called by Orchestrator on ActorType class before it has been spawned
-    
-    @classmethod
-    def get_running_actors(cls) -> list[int]:
-        """returns a list of pids of all running actors of this type"""
-        # WARNING: only works for process actors, not thread actors
-        if cls.mode == 'thread':
-            raise NotImplementedError('get_running_actors() is not implemented for thread actors')
-        return [
-            proc.pid for proc in cls._SPAWNED_ACTOR_PIDS
-            if proc.is_running() and proc.status() != 'zombie'
-        ]
-        
-    @classmethod
-    def get_actors_to_spawn(cls, queue: QuerySet, running_actors: list[int]) -> list[LaunchKwargs]:
-        """Get a list of launch kwargs for the number of actors to spawn based on the queue and currently running actors"""
-        queue_length = queue.count()
-        if not queue_length:                                      # queue is empty, spawn 0 actors
-            return []
-        
-        # WARNING:
-        # spawning new actors processes is slow/expensive, avoid spawning many actors at once in a single orchestrator tick.
-        # limit to spawning 1 or 2 at a time per orchestrator tick, and let the next tick handle starting another couple.
-        # DONT DO THIS:
-        # if queue_length > 20:                      # queue is extremely long, spawn maximum actors at once!
-        #   num_to_spawn_this_tick = cls.MAX_CONCURRENT_ACTORS
-        
-        if queue_length > 10:    
-            num_to_spawn_this_tick = 2  # spawn more actors per tick if queue is long
-        else:
-            num_to_spawn_this_tick = 1  # spawn fewer actors per tick if queue is short
-        
-        num_remaining = cls.MAX_CONCURRENT_ACTORS - len(running_actors)
-        num_to_spawn_now: int = limit(num_to_spawn_this_tick, num_remaining)
-        
-        actors_launch_kwargs: list[LaunchKwargs] = num_to_spawn_now * [{**cls.launch_kwargs}]
-        return actors_launch_kwargs
-        
-    @classmethod
-    def start(cls, mode: Literal['thread', 'process']='process', **launch_kwargs: LaunchKwargs) -> int:
-        if mode == 'thread':
-            return cls._fork_actor_as_thread(**launch_kwargs)
-        elif mode == 'process':
-            return cls._fork_actor_as_process(**launch_kwargs)
-        raise ValueError(f'Invalid actor mode: {mode} must be "thread" or "process"')
-    
-    @classproperty
-    def qs(cls) -> QuerySet[ModelType]:
-        """
-        Get the unfiltered and unsorted QuerySet of all objects that this Actor might care about.
-        Override this in the subclass to define the QuerySet of objects that the Actor is going to poll for new work.
-        (don't limit, order, or filter this by retry_at or status yet, Actor.get_queue() handles that part)
-        """
-        return cls.Model.objects.all()
-    
-    @classproperty
-    def final_q(cls) -> Q:
-        """Get the filter for objects that are in a final state"""
-        return Q(**{f'{cls.STATE_FIELD_NAME}__in': [cls._state_to_str(s) for s in cls.FINAL_STATES]})
-    
-    @classproperty
-    def active_q(cls) -> Q:
-        """Get the filter for objects that are actively processing right now"""
-        return Q(**{cls.STATE_FIELD_NAME: cls._state_to_str(cls.ACTIVE_STATE)})   # e.g. Q(status='started')
-    
-    @classproperty
-    def stalled_q(cls) -> Q:
-        """Get the filter for objects that are marked active but have timed out"""
-        return cls.active_q & Q(retry_at__lte=timezone.now())                     # e.g. Q(status='started') AND Q(<retry_at is in the past>)
-    
-    @classproperty
-    def future_q(cls) -> Q:
-        """Get the filter for objects that have a retry_at in the future"""
-        return Q(retry_at__gt=timezone.now())
-    
-    @classproperty
-    def pending_q(cls) -> Q:
-        """Get the filter for objects that are ready for processing."""
-        return ~(cls.active_q) & ~(cls.final_q) & ~(cls.future_q)
-    
-    @classmethod
-    def get_queue(cls, sort: bool=True) -> QuerySet[ModelType]:
-        """
-        Get the sorted and filtered QuerySet of objects that are ready for processing.
-        e.g. qs.exclude(status__in=('sealed', 'started'), retry_at__gt=timezone.now()).order_by('retry_at')
-        """
-        unsorted_qs = cls.qs.filter(cls.pending_q)
-        return unsorted_qs.order_by(*cls.CLAIM_ORDER) if sort else unsorted_qs
-
-    ### Instance Methods: Only called from within Actor instance after it has been spawned (i.e. forked as a thread or process)
-    
-    def runloop(self):
-        """The main runloop that starts running when the actor is spawned (as subprocess or thread) and exits when the queue is empty"""
-        self.on_startup()
-        obj_to_process: ModelType | None = None
-        last_error: BaseException | None = None
-        try:
-            while True:
-                # Get the next object to process from the queue
-                try:
-                    obj_to_process = cast(ModelType, self.get_next(atomic=self.atomic))
-                except (ActorQueueIsEmpty, ActorObjectAlreadyClaimed) as err:
-                    last_error = err
-                    obj_to_process = None
-                
-                # Handle the case where there is no next object to process
-                if obj_to_process:
-                    self.idle_count = 0   # reset idle count if we got an object
-                else:
-                    if self.idle_count >= 30:
-                        break             # stop looping and exit if queue is empty and we have idled for 30sec
-                    else:
-                        # print('Actor runloop()', f'pid={self.pid}', 'queue empty, rechecking...')
-                        self.idle_count += 1
-                        time.sleep(1)
-                        continue
-                
-                # Process the object by triggering its StateMachine.tick() method
-                self.on_tick_start(obj_to_process)
-                try:
-                    self.tick(obj_to_process)
-                except Exception as err:
-                    last_error = err
-                    # print(f'[red]🏃‍♂️ {self}.tick()[/red] {obj_to_process} ERROR: [red]{type(err).__name__}: {err}[/red]')
-                    db.connections.close_all()                         # always reset the db connection after an exception to clear any pending transactions
-                    self.on_tick_exception(obj_to_process, err)
-                finally:
-                    self.on_tick_end(obj_to_process)
-
-        except BaseException as err:
-            last_error = err
-            if isinstance(err, KeyboardInterrupt):
-                print()
-            else:
-                print(f'\n[red]🏃‍♂️ {self}.runloop() FATAL:[/red] {type(err).__name__}: {err}')
-                print(f'    Last processed object: {obj_to_process}')
-                raise
-        finally:
-            self.on_shutdown(last_obj=obj_to_process, last_error=last_error)
-    
-    def get_update_kwargs_to_claim_obj(self) -> dict[str, Any]:
-        """
-        Get the field values needed to mark an pending obj_to_process as being actively processing (aka claimed)
-        by the current Actor. returned kwargs will be applied using: qs.filter(id=obj_to_process.id).update(**kwargs).
-        F() expressions are allowed in field values if you need to update a field based on its current value.
-        Can be a defined as a normal method (instead of classmethod) on subclasses if it needs to access instance vars.
-        """
-        return {
-            self.STATE_FIELD_NAME: self.ACTIVE_STATE,
-            'retry_at': timezone.now() + timedelta(seconds=self.MAX_TICK_TIME),
-        }
-    
-    def get_next(self, atomic: bool | None=None) -> ModelType | None:
-        """get the next object from the queue, atomically locking it if self.atomic=True"""
-        atomic = self.CLAIM_ATOMIC if atomic is None else atomic
-        if atomic:
-            # fetch and claim the next object from in the queue in one go atomically
-            obj = self.get_next_atomic()
-        else:
-            # two-step claim: fetch the next object and lock it in a separate query
-            obj = self.get_next_non_atomic()
-        return obj
-    
-    def get_next_non_atomic(self) -> ModelType:
-        """
-        Naiively selects the top/first object from self.get_queue().order_by(*self.CLAIM_ORDER),
-        then claims it by running .update(status='started', retry_at=<now + MAX_TICK_TIME>).
-        
-        Do not use this method if there is more than one Actor racing to get objects from the same queue,
-        it will be slow/buggy as they'll compete to lock the same object at the same time (TOCTTOU race).
-        """
-        obj = self.get_queue().first()
-        if obj is None:
-            raise ActorQueueIsEmpty(f'No next object available in {self}.get_queue()')
-        
-        locked = self.get_queue().filter(id=obj.id).update(**self.get_update_kwargs_to_claim_obj())
-        if not locked:
-            raise ActorObjectAlreadyClaimed(f'Unable to lock the next {self.Model.__name__} object from {self}.get_queue().first()')
-        return obj
-        
-    def get_next_atomic(self) -> ModelType | None:
-        """
-        Selects the top n=50 objects from the queue and atomically claims a random one from that set.
-        This approach safely minimizes contention with other Actors trying to select from the same Queue.
-
-        The atomic query is roughly equivalent to the following:  (all done in one SQL query to avoid a TOCTTOU race)
-            top_candidates are selected from:   qs.order_by(*CLAIM_ORDER).only('id')[:CLAIM_FROM_TOP_N]
-            a single candidate is chosen using: qs.filter(id__in=top_n_candidates).order_by('?').first()
-            the chosen obj is claimed using:    qs.filter(id=chosen_obj).update(status=ACTIVE_STATE, retry_at=<now + MAX_TICK_TIME>)
-        """
-        # TODO: if we switch from SQLite to PostgreSQL in the future, we should change this
-        # to use SELECT FOR UPDATE instead of a subquery + ORDER BY RANDOM() LIMIT 1
-        
-        # e.g. SELECT id FROM core_archiveresult WHERE status NOT IN (...) AND retry_at <= '...' ORDER BY retry_at ASC LIMIT 50
-        qs = self.get_queue()
-        select_top_canidates_sql, select_params = self._sql_for_select_top_n_candidates(qs=qs)
-        assert select_top_canidates_sql.startswith('SELECT ')
-        
-        # e.g. UPDATE core_archiveresult SET status='%s', retry_at='%s' WHERE status NOT IN (...) AND retry_at <= '...'
-        update_claimed_obj_sql, update_params = self._sql_for_update_claimed_obj(qs=qs, update_kwargs=self.get_update_kwargs_to_claim_obj())
-        assert update_claimed_obj_sql.startswith('UPDATE ')
-        db_table = self.Model._meta.db_table  # e.g. core_archiveresult
-        
-        # subquery gets the pool of the top candidates e.g. self.get_queue().only('id')[:CLAIM_FROM_TOP_N]
-        # main query selects a random one from that pool, and claims it using .update(status=ACTIVE_STATE, retry_at=<now + MAX_TICK_TIME>)
-        # this is all done in one atomic SQL query to avoid TOCTTOU race conditions (as much as possible)
-        atomic_select_and_update_sql = f"""
-            {update_claimed_obj_sql} AND "{db_table}"."id" = (
-                SELECT "{db_table}"."id" FROM (
-                    {select_top_canidates_sql}
-                ) candidates
-                ORDER BY RANDOM()
-                LIMIT 1
-            )
-            RETURNING *;
-        """
-        try:
-            return self.Model.objects.raw(atomic_select_and_update_sql, (*update_params, *select_params))[0]
-        except KeyError:
-            if self.get_queue().exists():
-                raise ActorObjectAlreadyClaimed(f'Unable to lock the next {self.Model.__name__} object from {self}.get_queue().first()')
-            else:
-                raise ActorQueueIsEmpty(f'No next object available in {self}.get_queue()')
-
-    def tick(self, obj_to_process: ModelType) -> None:
-        """Call the object.sm.tick() method to process the object"""
-        print(f'[blue]🏃‍♂️ {self}.tick()[/blue] {obj_to_process}')
-        
-        # get the StateMachine instance from the object
-        obj_statemachine = self._get_state_machine_instance(obj_to_process)
-        
-        # trigger the event on the StateMachine instance
-        obj_tick_method = getattr(obj_statemachine, self.EVENT_NAME)  # e.g. obj_statemachine.tick()
-        obj_tick_method()
-        
-        # save the object to persist any state changes
-        obj_to_process.save()
-        
-    def on_startup(self) -> None:
-        if self.mode == 'thread':
-            self.pid = get_native_id()  # thread id
-            print(f'[green]🏃‍♂️ {self}.on_startup() STARTUP (THREAD)[/green]')
-        else:
-            self.pid = os.getpid()      # process id
-            print(f'[green]🏃‍♂️ {self}.on_startup() STARTUP (PROCESS)[/green]')
-        # abx.pm.hook.on_actor_startup(actor=self)
-        
-    def on_shutdown(self, last_obj: ModelType | None=None, last_error: BaseException | None=None) -> None:
-        if isinstance(last_error, KeyboardInterrupt) or last_error is None:
-            last_error_str = '[green](CTRL-C)[/green]'
-        elif isinstance(last_error, ActorQueueIsEmpty):
-            last_error_str = '[green](queue empty)[/green]'
-        elif isinstance(last_error, ActorObjectAlreadyClaimed):
-            last_error_str = '[green](queue race)[/green]'
-        else:
-            last_error_str = f'[red]{type(last_error).__name__}: {last_error}[/red]'
-
-        print(f'[grey53]🏃‍♂️ {self}.on_shutdown() SHUTTING DOWN[/grey53] {last_error_str}')
-        # abx.pm.hook.on_actor_shutdown(actor=self, last_obj=last_obj, last_error=last_error)
-        
-    def on_tick_start(self, obj_to_process: ModelType) -> None:
-        print(f'🏃‍♂️ {self}.on_tick_start() {obj_to_process}')
-        # abx.pm.hook.on_actor_tick_start(actor=self, obj_to_process=obj)
-        # self.timer = TimedProgress(self.MAX_TICK_TIME, prefix='      ')
-    
-    def on_tick_end(self, obj_to_process: ModelType) -> None:
-        print(f'🏃‍♂️ {self}.on_tick_end() {obj_to_process}')
-        # abx.pm.hook.on_actor_tick_end(actor=self, obj_to_process=obj_to_process)
-        # self.timer.end()
-    
-    def on_tick_exception(self, obj_to_process: ModelType, error: Exception) -> None:
-        print(f'[red]🏃‍♂️ {self}.on_tick_exception()[/red] {obj_to_process}: [red]{type(error).__name__}: {error}[/red]')
-        # abx.pm.hook.on_actor_tick_exception(actor=self, obj_to_process=obj_to_process, error=error)
-
-
-
-def compile_sql_select(queryset: QuerySet, filter_kwargs: dict[str, Any] | None=None, order_args: tuple[str, ...]=(), limit: int | None=None) -> tuple[str, tuple[Any, ...]]:
-    """
-    Compute the SELECT query SQL for a queryset.filter(**filter_kwargs).order_by(*order_args)[:limit] call
-    Returns a tuple of (sql, params) where sql is a template string containing %s (unquoted) placeholders for the params
-    
-    WARNING:
-    final_sql = sql % params  DOES NOT WORK to assemble the final SQL string because the %s placeholders are not quoted/escaped
-    they should always passed separately to the DB driver so it can do its own quoting/escaping to avoid SQL injection and syntax errors
-    """
-    assert isinstance(queryset, QuerySet), f'compile_sql_select(...) first argument must be a QuerySet, got: {type(queryset).__name__} instead'
-    assert filter_kwargs is None or isinstance(filter_kwargs, dict), f'compile_sql_select(...) filter_kwargs argument must be a dict[str, Any], got: {type(filter_kwargs).__name__} instead'
-    assert isinstance(order_args, tuple) and all(isinstance(arg, str) for arg in order_args), f'compile_sql_select(...) order_args argument must be a tuple[str, ...] got: {type(order_args).__name__} instead'
-    assert limit is None or isinstance(limit, int), f'compile_sql_select(...) limit argument must be an int, got: {type(limit).__name__} instead'
-    
-    queryset = queryset._chain()                      # type: ignore   # copy queryset to avoid modifying the original
-    if filter_kwargs:
-        queryset = queryset.filter(**filter_kwargs)
-    if order_args:
-        queryset = queryset.order_by(*order_args)
-    if limit is not None:
-        queryset = queryset[:limit]
-    query = queryset.query
-    
-    # e.g. SELECT id FROM core_archiveresult WHERE status NOT IN (%s, %s, %s) AND retry_at <= %s ORDER BY retry_at ASC LIMIT 50
-    select_sql, select_params = query.get_compiler(queryset.db).as_sql()
-    return select_sql, select_params
-
-
-def compile_sql_update(queryset: QuerySet, update_kwargs: dict[str, Any], filter_kwargs: dict[str, Any] | None=None) -> tuple[str, tuple[Any, ...]]:
-    """
-    Compute the UPDATE query SQL for a queryset.filter(**filter_kwargs).update(**update_kwargs) call
-    Returns a tuple of (sql, params) where sql is a template string containing %s (unquoted) placeholders for the params
-    
-    Based on the django.db.models.QuerySet.update() source code, but modified to return the SQL instead of executing the update
-    https://github.com/django/django/blob/611bf6c2e2a1b4ab93273980c45150c099ab146d/django/db/models/query.py#L1217
-    
-    WARNING:
-    final_sql = sql % params  DOES NOT WORK to assemble the final SQL string because the %s placeholders are not quoted/escaped
-    they should always passed separately to the DB driver so it can do its own quoting/escaping to avoid SQL injection and syntax errors
-    """
-    assert isinstance(queryset, QuerySet), f'compile_sql_update(...) first argument must be a QuerySet, got: {type(queryset).__name__} instead'
-    assert isinstance(update_kwargs, dict), f'compile_sql_update(...) update_kwargs argument must be a dict[str, Any], got: {type(update_kwargs).__name__} instead'
-    assert filter_kwargs is None or isinstance(filter_kwargs, dict), f'compile_sql_update(...) filter_kwargs argument must be a dict[str, Any], got: {type(filter_kwargs).__name__} instead'
-    
-    queryset = queryset._chain()                      # type: ignore   # copy queryset to avoid modifying the original
-    if filter_kwargs:
-        queryset = queryset.filter(**filter_kwargs)
-    queryset.query.clear_ordering(force=True)                          # clear any ORDER BY clauses
-    queryset.query.clear_limits()                                      # clear any LIMIT clauses aka slices[:n]
-    queryset._for_write = True                        # type: ignore
-    query = queryset.query.chain(sql.UpdateQuery)     # type: ignore
-    query.add_update_values(update_kwargs)            # type: ignore
-    query.annotations = {}                                             # clear any annotations
-    
-    # e.g. UPDATE core_archiveresult SET status='%s', retry_at='%s' WHERE status NOT IN (%s, %s, %s) AND retry_at <= %s
-    update_sql, update_params = query.get_compiler(queryset.db).as_sql()
-    return update_sql, update_params
@@ -1,6 +0,0 @@
-from django.apps import AppConfig
-
-
-class ActorsConfig(AppConfig):
-    default_auto_field = "django.db.models.BigAutoField"
-    name = "actors"
@@ -1,300 +0,0 @@
-from typing import ClassVar, Type, Iterable
-from datetime import datetime, timedelta
-
-from statemachine.mixins import MachineMixin
-
-from django.db import models
-from django.utils import timezone
-from django.utils.functional import classproperty
-
-from statemachine import registry, StateMachine, State
-
-from django.core import checks
-
-class DefaultStatusChoices(models.TextChoices):
-    QUEUED = 'queued', 'Queued'
-    STARTED = 'started', 'Started'
-    SEALED = 'sealed', 'Sealed'
-
-
-default_status_field: models.CharField = models.CharField(choices=DefaultStatusChoices.choices, max_length=15, default=DefaultStatusChoices.QUEUED, null=False, blank=False, db_index=True)
-default_retry_at_field: models.DateTimeField = models.DateTimeField(default=timezone.now, null=False, db_index=True)
-
-ObjectState = State | str
-ObjectStateList = Iterable[ObjectState]
-
-
-class BaseModelWithStateMachine(models.Model, MachineMixin):
-    id: models.UUIDField
-    
-    StatusChoices: ClassVar[Type[models.TextChoices]]
-    
-    # status: models.CharField
-    # retry_at: models.DateTimeField
-    
-    state_machine_name: ClassVar[str]
-    state_field_name: ClassVar[str]
-    state_machine_attr: ClassVar[str] = 'sm'
-    bind_events_as_methods: ClassVar[bool] = True
-    
-    active_state: ClassVar[ObjectState]
-    retry_at_field_name: ClassVar[str]
-    
-    class Meta:
-        abstract = True
-        
-    @classmethod
-    def check(cls, sender=None, **kwargs):
-        errors = super().check(**kwargs)
-        
-        found_id_field = False
-        found_status_field = False
-        found_retry_at_field = False
-        
-        for field in cls._meta.get_fields():
-            if getattr(field, '_is_state_field', False):
-                if cls.state_field_name == field.name:
-                    found_status_field = True
-                    if getattr(field, 'choices', None) != cls.StatusChoices.choices:
-                        errors.append(checks.Error(
-                            f'{cls.__name__}.{field.name} must have choices set to {cls.__name__}.StatusChoices.choices',
-                            hint=f'{cls.__name__}.{field.name}.choices = {getattr(field, "choices", None)!r}',
-                            obj=cls,
-                            id='actors.E011',
-                        ))
-            if getattr(field, '_is_retry_at_field', False):
-                if cls.retry_at_field_name == field.name:
-                    found_retry_at_field = True
-            if field.name == 'id' and getattr(field, 'primary_key', False):
-                found_id_field = True
-                    
-        if not found_status_field:
-            errors.append(checks.Error(
-                f'{cls.__name__}.state_field_name must be defined and point to a StatusField()',
-                hint=f'{cls.__name__}.state_field_name = {cls.state_field_name!r} but {cls.__name__}.{cls.state_field_name!r} was not found or does not refer to StatusField',
-                obj=cls,
-                id='actors.E012',
-            ))
-        if not found_retry_at_field:
-            errors.append(checks.Error(
-                f'{cls.__name__}.retry_at_field_name must be defined and point to a RetryAtField()',
-                hint=f'{cls.__name__}.retry_at_field_name = {cls.retry_at_field_name!r} but {cls.__name__}.{cls.retry_at_field_name!r} was not found or does not refer to RetryAtField',
-                obj=cls,
-                id='actors.E013',
-            ))
-            
-        if not found_id_field:
-            errors.append(checks.Error(
-                f'{cls.__name__} must have an id field that is a primary key',
-                hint=f'{cls.__name__}.id = {cls.id!r}',
-                obj=cls,
-                id='actors.E014',
-            ))
-            
-        if not isinstance(cls.state_machine_name, str):
-            errors.append(checks.Error(
-                f'{cls.__name__}.state_machine_name must be a dotted-import path to a StateMachine class',
-                hint=f'{cls.__name__}.state_machine_name = {cls.state_machine_name!r}',
-                obj=cls,
-                id='actors.E015',
-            ))
-        
-        try:
-            cls.StateMachineClass
-        except Exception as err:
-            errors.append(checks.Error(
-                f'{cls.__name__}.state_machine_name must point to a valid StateMachine class, but got {type(err).__name__} {err} when trying to access {cls.__name__}.StateMachineClass',
-                hint=f'{cls.__name__}.state_machine_name = {cls.state_machine_name!r}',
-                obj=cls,
-                id='actors.E016',
-            ))
-        
-        if cls.INITIAL_STATE not in cls.StatusChoices.values:
-            errors.append(checks.Error(
-                f'{cls.__name__}.StateMachineClass.initial_state must be present within {cls.__name__}.StatusChoices',
-                hint=f'{cls.__name__}.StateMachineClass.initial_state = {cls.StateMachineClass.initial_state!r}',
-                obj=cls,
-                id='actors.E017',
-            ))
-            
-        if cls.ACTIVE_STATE not in cls.StatusChoices.values:
-            errors.append(checks.Error(
-                f'{cls.__name__}.active_state must be set to a valid State present within {cls.__name__}.StatusChoices',
-                hint=f'{cls.__name__}.active_state = {cls.active_state!r}',
-                obj=cls,
-                id='actors.E018',
-            ))
-            
-        
-        for state in cls.FINAL_STATES:
-            if state not in cls.StatusChoices.values:
-                errors.append(checks.Error(
-                    f'{cls.__name__}.StateMachineClass.final_states must all be present within {cls.__name__}.StatusChoices',
-                    hint=f'{cls.__name__}.StateMachineClass.final_states = {cls.StateMachineClass.final_states!r}',
-                    obj=cls,
-                    id='actors.E019',
-                ))
-                break
-        return errors
-    
-    @staticmethod
-    def _state_to_str(state: ObjectState) -> str:
-        """Convert a statemachine.State, models.TextChoices.choices value, or Enum value to a str"""
-        return str(state.value) if isinstance(state, State) else str(state)
-    
-    
-    @property
-    def RETRY_AT(self) -> datetime:
-        return getattr(self, self.retry_at_field_name)
-    
-    @RETRY_AT.setter
-    def RETRY_AT(self, value: datetime):
-        setattr(self, self.retry_at_field_name, value)
-        
-    @property
-    def STATE(self) -> str:
-        return getattr(self, self.state_field_name)
-    
-    @STATE.setter
-    def STATE(self, value: str):
-        setattr(self, self.state_field_name, value)
-        
-    def bump_retry_at(self, seconds: int = 10):
-        self.RETRY_AT = timezone.now() + timedelta(seconds=seconds)
-        
-    @classproperty
-    def ACTIVE_STATE(cls) -> str:
-        return cls._state_to_str(cls.active_state)
-        
-    @classproperty
-    def INITIAL_STATE(cls) -> str:
-        return cls._state_to_str(cls.StateMachineClass.initial_state)
-    
-    @classproperty
-    def FINAL_STATES(cls) -> list[str]:
-        return [cls._state_to_str(state) for state in cls.StateMachineClass.final_states]
-    
-    @classproperty
-    def FINAL_OR_ACTIVE_STATES(cls) -> list[str]:
-        return [*cls.FINAL_STATES, cls.ACTIVE_STATE]
-        
-    @classmethod
-    def extend_choices(cls, base_choices: Type[models.TextChoices]):
-        """
-        Decorator to extend the base choices with extra choices, e.g.:
-        
-        class MyModel(ModelWithStateMachine):
-        
-            @ModelWithStateMachine.extend_choices(ModelWithStateMachine.StatusChoices)
-            class StatusChoices(models.TextChoices):
-                SUCCEEDED = 'succeeded'
-                FAILED = 'failed'
-                SKIPPED = 'skipped'
-        """
-        assert issubclass(base_choices, models.TextChoices), f'@extend_choices(base_choices) must be a TextChoices class, not {base_choices.__name__}'
-        def wrapper(extra_choices: Type[models.TextChoices]) -> Type[models.TextChoices]:
-            joined = {}
-            for item in base_choices.choices:
-                joined[item[0]] = item[1]
-            for item in extra_choices.choices:
-                joined[item[0]] = item[1]
-            return models.TextChoices('StatusChoices', joined)
-        return wrapper
-        
-    @classmethod
-    def StatusField(cls, **kwargs) -> models.CharField:
-        """
-        Used on subclasses to extend/modify the status field with updated kwargs. e.g.:
-        
-        class MyModel(ModelWithStateMachine):
-            class StatusChoices(ModelWithStateMachine.StatusChoices):
-                QUEUED = 'queued', 'Queued'
-                STARTED = 'started', 'Started'
-                SEALED = 'sealed', 'Sealed'
-                BACKOFF = 'backoff', 'Backoff'
-                FAILED = 'failed', 'Failed'
-                SKIPPED = 'skipped', 'Skipped'
-        
-            status = ModelWithStateMachine.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED)
-        """
-        default_kwargs = default_status_field.deconstruct()[3]
-        updated_kwargs = {**default_kwargs, **kwargs}
-        field = models.CharField(**updated_kwargs)
-        field._is_state_field = True                    # type: ignore
-        return field
-
-    @classmethod
-    def RetryAtField(cls, **kwargs) -> models.DateTimeField:
-        """
-        Used on subclasses to extend/modify the retry_at field with updated kwargs. e.g.:
-        
-        class MyModel(ModelWithStateMachine):
-            retry_at = ModelWithStateMachine.RetryAtField(editable=False)
-        """
-        default_kwargs = default_retry_at_field.deconstruct()[3]
-        updated_kwargs = {**default_kwargs, **kwargs}
-        field = models.DateTimeField(**updated_kwargs)
-        field._is_retry_at_field = True                 # type: ignore
-        return field
-    
-    @classproperty
-    def StateMachineClass(cls) -> Type[StateMachine]:
-        """Get the StateMachine class for the given django Model that inherits from MachineMixin"""
-
-        model_state_machine_name = getattr(cls, 'state_machine_name', None)
-        if model_state_machine_name:
-            StateMachineCls = registry.get_machine_cls(model_state_machine_name)
-            assert issubclass(StateMachineCls, StateMachine)
-            return StateMachineCls
-        raise NotImplementedError(f'ActorType[{cls.__name__}] must define .state_machine_name: str that points to a valid StateMachine')
-    
-    # @classproperty
-    # def final_q(cls) -> Q:
-    #     """Get the filter for objects that are in a final state"""
-    #     return Q(**{f'{cls.state_field_name}__in': cls.final_states})
-    
-    # @classproperty
-    # def active_q(cls) -> Q:
-    #     """Get the filter for objects that are actively processing right now"""
-    #     return Q(**{cls.state_field_name: cls._state_to_str(cls.active_state)})   # e.g. Q(status='started')
-    
-    # @classproperty
-    # def stalled_q(cls) -> Q:
-    #     """Get the filter for objects that are marked active but have timed out"""
-    #     return cls.active_q & Q(retry_at__lte=timezone.now())                     # e.g. Q(status='started') AND Q(<retry_at is in the past>)
-    
-    # @classproperty
-    # def future_q(cls) -> Q:
-    #     """Get the filter for objects that have a retry_at in the future"""
-    #     return Q(retry_at__gt=timezone.now())
-    
-    # @classproperty
-    # def pending_q(cls) -> Q:
-    #     """Get the filter for objects that are ready for processing."""
-    #     return ~(cls.active_q) & ~(cls.final_q) & ~(cls.future_q)
-    
-    # @classmethod
-    # def get_queue(cls) -> QuerySet:
-    #     """
-    #     Get the sorted and filtered QuerySet of objects that are ready for processing.
-    #     e.g. qs.exclude(status__in=('sealed', 'started'), retry_at__gt=timezone.now()).order_by('retry_at')
-    #     """
-    #     return cls.objects.filter(cls.pending_q)
-
-
-class ModelWithStateMachine(BaseModelWithStateMachine):
-    StatusChoices: ClassVar[Type[DefaultStatusChoices]] = DefaultStatusChoices
-    
-    status: models.CharField = BaseModelWithStateMachine.StatusField()
-    retry_at: models.DateTimeField = BaseModelWithStateMachine.RetryAtField()
-    
-    state_machine_name: ClassVar[str]      # e.g. 'core.statemachines.ArchiveResultMachine'
-    state_field_name: ClassVar[str]        = 'status'
-    state_machine_attr: ClassVar[str]      = 'sm'
-    bind_events_as_methods: ClassVar[bool] = True
-    
-    active_state: ClassVar[str]            = StatusChoices.STARTED
-    retry_at_field_name: ClassVar[str]     = 'retry_at'
-    
-    class Meta:
-        abstract = True
@@ -1,244 +0,0 @@
-__package__ = 'archivebox.actors'
-
-import os
-import time
-import itertools
-from typing import Dict, Type, Literal, ClassVar
-from django.utils.functional import classproperty
-
-from multiprocessing import Process
-from threading import Thread, get_native_id
-
-
-from rich import print
-
-from django.db.models import QuerySet
-
-from django.apps import apps
-from .actor import ActorType
-
-class Orchestrator:
-    pid: int
-    idle_count: int = 0
-    actor_types: Dict[str, Type[ActorType]]
-    mode: Literal['thread', 'process'] = 'process'
-
-    def __init__(self, actor_types: Dict[str, Type[ActorType]] | None = None, mode: Literal['thread', 'process'] | None=None):
-        self.actor_types = actor_types or self.actor_types or self.autodiscover_actor_types()
-        self.mode = mode or self.mode
-
-    def __repr__(self) -> str:
-        label = 'tid' if self.mode == 'thread' else 'pid'
-        return f'[underline]{self.name}[/underline]\\[{label}={self.pid}]'
-    
-    def __str__(self) -> str:
-        return self.__repr__()
-    
-    @classproperty
-    def name(cls) -> str:
-        return cls.__name__   # type: ignore
-    
-    def fork_as_thread(self):
-        self.thread = Thread(target=self.runloop)
-        self.thread.start()
-        assert self.thread.native_id is not None
-        return self.thread.native_id
-    
-    def fork_as_process(self):
-        self.process = Process(target=self.runloop)
-        self.process.start()
-        assert self.process.pid is not None
-        return self.process.pid
-
-    def start(self) -> int:
-        if self.mode == 'thread':
-            return self.fork_as_thread()
-        elif self.mode == 'process':
-            return self.fork_as_process()
-        raise ValueError(f'Invalid orchestrator mode: {self.mode}')
-    
-    @classmethod
-    def autodiscover_actor_types(cls) -> Dict[str, Type[ActorType]]:
-        # returns a Dict of all discovered {actor_type_id: ActorType} across the codebase
-        # override this method in a subclass to customize the actor types that are used
-        # return {'Snapshot': SnapshotActorType, 'ArchiveResult_chrome': ChromeActorType, ...}
-        return {
-            # look through all models and find all classes that inherit from ActorType
-            # actor_type.__name__: actor_type
-            # for actor_type in abx.pm.hook.get_all_ACTORS_TYPES().values()
-        }
-    
-    @classmethod
-    def get_orphaned_objects(cls, all_queues) -> list:
-        # returns a list of objects that are in the queues of all actor types but not in the queues of any other actor types
-        all_queued_ids = itertools.chain(*[queue.values('id', flat=True) for queue in all_queues.values()])
-        orphaned_objects = []
-        for model in apps.get_models():
-            if hasattr(model, 'retry_at'):
-                orphaned_objects.extend(model.objects.filter(retry_at__lt=timezone.now()).exclude(id__in=all_queued_ids))
-        return orphaned_objects
-    
-    def on_startup(self):
-        if self.mode == 'thread':
-            self.pid = get_native_id()
-            print(f'[green]👨‍✈️ {self}.on_startup() STARTUP (THREAD)[/green]')
-        elif self.mode == 'process':
-            self.pid = os.getpid()
-            print(f'[green]👨‍✈️ {self}.on_startup() STARTUP (PROCESS)[/green]')
-        # abx.pm.hook.on_orchestrator_startup(self)
-    
-    def on_shutdown(self, err: BaseException | None = None):
-        print(f'[grey53]👨‍✈️ {self}.on_shutdown() SHUTTING DOWN[/grey53]', err or '[green](gracefully)[/green]')
-        # abx.pm.hook.on_orchestrator_shutdown(self)
-        
-    def on_tick_started(self, all_queues):
-        # total_pending = sum(queue.count() for queue in all_queues.values())
-        # print(f'👨‍✈️ {self}.on_tick_started()', f'total_pending={total_pending}')
-        # abx.pm.hook.on_orchestrator_tick_started(self, actor_types, all_queues)
-        pass
-    
-    def on_tick_finished(self, all_queues, all_existing_actors, all_spawned_actors):
-        if all_spawned_actors:
-            total_queue_length = sum(queue.count() for queue in all_queues.values())
-            print(f'[grey53]👨‍✈️ {self}.on_tick_finished() queue={total_queue_length} existing_actors={len(all_existing_actors)} spawned_actors={len(all_spawned_actors)}[/grey53]')
-        # abx.pm.hook.on_orchestrator_tick_finished(self, actor_types, all_queues)
-
-    def on_idle(self, all_queues):
-        # print(f'👨‍✈️ {self}.on_idle()')
-        # abx.pm.hook.on_orchestrator_idle(self)
-        # check for orphaned objects left behind
-        if self.idle_count == 60:
-            orphaned_objects = self.get_orphaned_objects(all_queues)
-            if orphaned_objects:
-                print('[red]👨‍✈️ WARNING: some objects may not be processed, no actor has claimed them after 60s:[/red]', orphaned_objects)
-
-    def runloop(self):
-        self.on_startup()
-        try:
-            while True:
-                all_queues = {
-                    actor_type: actor_type.get_queue()
-                    for actor_type in self.actor_types.values()
-                }
-                if not all_queues:
-                    raise Exception('Failed to find any actor_types to process')
-
-                self.on_tick_started(all_queues)
-
-                all_existing_actors = []
-                all_spawned_actors = []
-
-                for actor_type, queue in all_queues.items():
-                    try:
-                        existing_actors = actor_type.get_running_actors()
-                        all_existing_actors.extend(existing_actors)
-                        actors_to_spawn = actor_type.get_actors_to_spawn(queue, existing_actors)
-                        for launch_kwargs in actors_to_spawn:
-                            new_actor_pid = actor_type.start(mode='process', **launch_kwargs)
-                            all_spawned_actors.append(new_actor_pid)
-                    except Exception as err:
-                        print(f'🏃‍♂️ ERROR: {self} Failed to get {actor_type} queue & running actors', err)
-                    except BaseException:
-                        raise
-
-                if not any(queue.exists() for queue in all_queues.values()):
-                    self.on_idle(all_queues)
-                    self.idle_count += 1
-                    time.sleep(1)
-                else:
-                    self.idle_count = 0
-                    
-                self.on_tick_finished(all_queues, all_existing_actors, all_spawned_actors)
-                time.sleep(1)
-
-        except BaseException as err:
-            if isinstance(err, KeyboardInterrupt):
-                print()
-            else:
-                print(f'\n[red]🏃‍♂️ {self}.runloop() FATAL:[/red]', err.__class__.__name__, err)
-            self.on_shutdown(err=err)
-
-
-
-from archivebox.config.django import setup_django
-
-setup_django()
-
-from core.models import ArchiveResult, Snapshot
-
-from django.utils import timezone
-
-from django import db
-from django.db import connection
-
-
-from crawls.actors import CrawlActor
-from .actor_snapshot import SnapshotActor
-
-from abx_plugin_singlefile.actors import SinglefileActor
-
-
-class FaviconActor(ActorType[ArchiveResult]):
-    CLAIM_ORDER: ClassVar[str] = 'created_at DESC'
-    CLAIM_WHERE: ClassVar[str] = 'status = "queued" AND extractor = "favicon"'
-    CLAIM_SET: ClassVar[str] = 'status = "started"'
-    
-    @classproperty
-    def QUERYSET(cls) -> QuerySet:
-        return ArchiveResult.objects.filter(status='failed', extractor='favicon')
-
-    def tick(self, obj: ArchiveResult):
-        print(f'[grey53]{self}.tick({obj.abid or obj.id}, status={obj.status}) remaining:[/grey53]', self.get_queue().count())
-        updated = ArchiveResult.objects.filter(id=obj.id, status='started').update(status='success') == 1
-        if not updated:
-            raise Exception(f'Failed to update {obj.abid or obj.id}, interrupted by another actor writing to the same object')
-        obj.refresh_from_db()
-        obj.save()
-
-
-class ExtractorsOrchestrator(Orchestrator):
-    actor_types = {
-        'CrawlActor': CrawlActor,
-        'SnapshotActor': SnapshotActor,
-        'FaviconActor': FaviconActor,
-        'SinglefileActor': SinglefileActor,
-    }
-
-
-if __name__ == '__main__':    
-    orchestrator = ExtractorsOrchestrator()
-    orchestrator.start()
-    
-    snap = Snapshot.objects.last()
-    assert snap is not None
-    created = 0
-    while True:
-        time.sleep(0.05)
-        # try:
-        #     ArchiveResult.objects.bulk_create([
-        #         ArchiveResult(
-        #             id=uuid.uuid4(),
-        #             snapshot=snap,
-        #             status='failed',
-        #             extractor='favicon',
-        #             cmd=['echo', '"hello"'],
-        #             cmd_version='1.0',
-        #             pwd='.',
-        #             start_ts=timezone.now(),
-        #             end_ts=timezone.now(),
-        #             created_at=timezone.now(),
-        #             modified_at=timezone.now(),
-        #             created_by_id=1,
-        #         )
-        #         for _ in range(100)
-        #     ])
-        #     created += 100
-        #     if created % 1000 == 0:
-        #         print(f'[blue]Created {created} ArchiveResults...[/blue]')
-        #         time.sleep(25)
-        # except Exception as err:
-        #     print(err)
-        #     db.connections.close_all()
-        # except BaseException as err:
-        #     print(err)
-        #     break
@@ -1,31 +0,0 @@
-__package__ = 'archivebox.api'
-
-from signal_webhooks.admin import WebhookAdmin
-from signal_webhooks.utils import get_webhook_model
-
-from abid_utils.admin import ABIDModelAdmin
-
-from api.models import APIToken
-
-
-class APITokenAdmin(ABIDModelAdmin):
-    list_display = ('created_at', 'abid', 'created_by', 'token_redacted', 'expires')
-    sort_fields = ('abid', 'created_at', 'created_by', 'expires')
-    readonly_fields = ('created_at', 'modified_at', 'abid_info')
-    search_fields = ('id', 'abid', 'created_by__username', 'token')
-    fields = ('created_by', 'token', 'expires', *readonly_fields)
-
-    list_filter = ('created_by',)
-    ordering = ['-created_at']
-    list_per_page = 100
-
-
-class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
-    list_display = ('created_at', 'created_by', 'abid', *WebhookAdmin.list_display)
-    sort_fields = ('created_at', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
-    readonly_fields = ('created_at', 'modified_at', 'abid_info', *WebhookAdmin.readonly_fields)
-
-
-def register_admin(admin_site):
-    admin_site.register(APIToken, APITokenAdmin)
-    admin_site.register(get_webhook_model(), CustomWebhookAdmin)
@@ -2,14 +2,10 @@ __package__ = 'archivebox.api'

 from django.apps import AppConfig

-import abx


 class APIConfig(AppConfig):
    name = 'api'

-
-@abx.hookimpl
-def register_admin(admin_site):
-    from api.admin import register_admin
-    register_admin(admin_site)
+    def ready(self):
+        pass
@@ -1,34 +1,13 @@
 __package__ = 'archivebox.api'

-from typing import Any, Optional, cast
-from datetime import timedelta
+from typing import Optional, cast

 from django.http import HttpRequest
-from django.utils import timezone
 from django.contrib.auth import login
 from django.contrib.auth import authenticate
 from django.contrib.auth.models import AbstractBaseUser

 from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser
-from ninja.errors import HttpError
-
-
-def get_or_create_api_token(user):
-    from api.models import APIToken
-    
-    if user and user.is_superuser:
-        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
-        if api_tokens.exists():
-            # unexpired token exists, use it
-            api_token = api_tokens.last()
-        else:
-            # does not exist, create a new one
-            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
-        
-        assert api_token.is_valid(), f"API token is not valid {api_token}"
-
-        return api_token
-    return None


 def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
@@ -37,35 +16,39 @@ def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[Abs
    
    user = None

-    submitted_empty_form = str(token).strip() in ('string', '', 'None', 'null')
-    if not submitted_empty_form:
+    submitted_empty_form = token in ('string', '', None)
+    if submitted_empty_form:
+        assert request is not None, 'No request provided for API key authentication'
+        user = request.user       # see if user is authed via django session and use that as the default
+    else:
        try:
            token = APIToken.objects.get(token=token)
            if token.is_valid():
                user = token.created_by
-                request._api_token = token
        except APIToken.DoesNotExist:
            pass

    if not user:
-        # print('[❌] Failed to authenticate API user using API Key:', request)
-        return None
-    
-    return cast(AbstractBaseUser, user)
+        print('[❌] Failed to authenticate API user using API Key:', request)
+
+    return None

 def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
    """Given a username and password, check if they are valid and return the corresponding user"""
    user = None
    
    submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
-    if not submitted_empty_form:
+    if submitted_empty_form:
+        assert request is not None, 'No request provided for API key authentication'
+        user = request.user       # see if user is authed via django session and use that as the default
+    else:
        user = authenticate(
            username=username,
            password=password,
        )

    if not user:
-        # print('[❌] Failed to authenticate API user using API Key:', request)
+        print('[❌] Failed to authenticate API user using API Key:', request)
        user = None

    return cast(AbstractBaseUser | None, user)
@@ -73,41 +56,28 @@ def auth_using_password(username, password, request: Optional[HttpRequest]=None)

 ### Base Auth Types

-
 class APITokenAuthCheck:
    """The base class for authentication methods that use an api.models.APIToken"""
    def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        request.user = auth_using_token(
+        user = auth_using_token(
            token=key,
            request=request,
        )
-        if request.user and request.user.pk:
-            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
-            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
-            request._api_auth_method = self.__class__.__name__
-
-            if not request.user.is_superuser:
-                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
-        return request.user
-
+        if user is not None:
+            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
+        return user

 class UserPassAuthCheck:
    """The base class for authentication methods that use a username & password"""
    def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
-        request.user = auth_using_password(
+        user = auth_using_password(
            username=username,
            password=password,
            request=request,
        )
-        if request.user and request.user.pk:
-            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
-            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
-            request._api_auth_method = self.__class__.__name__
-
-            if not request.user.is_superuser:
-                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
-
-        return request.user
+        if user is not None:
+            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
+        return user


 ### Django-Ninja-Provided Auth Methods
@@ -128,12 +98,13 @@ class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
    pass

+
 ### Enabled Auth Methods

 API_AUTH_METHODS = [
    HeaderTokenAuth(),
    BearerTokenAuth(),
    QueryParamTokenAuth(), 
-    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
+    django_auth_superuser,
    UsernameAndPasswordAuth(),
 ]
@@ -1,47 +0,0 @@
-# Generated by Django 5.1 on 2024-09-04 23:32
-
-import abid_utils.models
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0007_alter_apitoken_created_by'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created',
-            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created_by',
-            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created',
-            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created_by',
-            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='id',
-            field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
-        ),
-    ]
@@ -1,39 +0,0 @@
-# Generated by Django 5.1 on 2024-09-05 00:26
-
-import abid_utils.models
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0008_alter_apitoken_created_alter_apitoken_created_by_and_more'),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='created',
-            new_name='created_at',
-        ),
-        migrations.RenameField(
-            model_name='apitoken',
-            old_name='modified',
-            new_name='modified_at',
-        ),
-        migrations.RenameField(
-            model_name='outboundwebhook',
-            old_name='modified',
-            new_name='modified_at',
-        ),
-        migrations.AddField(
-            model_name='outboundwebhook',
-            name='created_at',
-            field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created',
-            field=models.DateTimeField(auto_now_add=True, help_text='When the webhook was created.', verbose_name='created'),
-        ),
-    ]
@@ -1,5 +1,6 @@
 __package__ = 'archivebox.api'

+import uuid
 import secrets
 from datetime import timedelta

@@ -11,7 +12,7 @@ from signal_webhooks.models import WebhookBase

 from django_stubs_ext.db.models import TypedModelMeta

-from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
+from abid_utils.models import ABIDModel, ABIDField, get_or_create_system_user_pk



@@ -26,18 +27,16 @@ class APIToken(ABIDModel):
    """
    # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
    abid_prefix = 'apt_'
-    abid_ts_src = 'self.created_at'
-    abid_uri_src = 'self.created_by_id'
-    abid_subtype_src = '"01"'
+    abid_ts_src = 'self.created'
+    abid_uri_src = 'self.token'
+    abid_subtype_src = 'self.created_by_id'
    abid_rand_src = 'self.id'
-    abid_drift_allowed = True

-    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    abid = ABIDField(prefix=abid_prefix)

-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    created = models.DateTimeField(auto_now_add=True)

    token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
    expires = models.DateTimeField(null=True, blank=True)
@@ -59,7 +58,7 @@ class APIToken(ABIDModel):
            "abid":             str(self.ABID),
            "created_by_id":    str(self.created_by_id),
            "token":            self.token,
-            "created_at":       self.created_at.isoformat(),
+            "created":          self.created.isoformat(),
            "expires":          self.expires_as_iso8601,
        }

@@ -95,18 +94,17 @@ class OutboundWebhook(ABIDModel, WebhookBase):
        settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
    """
    abid_prefix = 'whk_'
-    abid_ts_src = 'self.created_at'
+    abid_ts_src = 'self.created'
    abid_uri_src = 'self.endpoint'
    abid_subtype_src = 'self.ref'
    abid_rand_src = 'self.id'
-    abid_drift_allowed = True

-    id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    abid = ABIDField(prefix=abid_prefix)

-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False)
-    created_at = AutoDateTimeField(default=None, null=False, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)

    # More fields here: WebhookBase...

@@ -122,6 +120,3 @@ class OutboundWebhook(ABIDModel, WebhookBase):
    class Meta(WebhookBase.Meta):
        verbose_name = 'API Outbound Webhook'

-
-    def __str__(self) -> str:
-        return f'[{self.abid}] {self.ref} -> {self.endpoint}'
@@ -1,30 +1,30 @@
 __package__ = 'archivebox.api'

-# from django.test import TestCase
-# from ninja.testing import TestClient
+from django.test import TestCase
+from ninja.testing import TestClient

-# from .routes_cli import router
+from .routes_cli import router

-# class ArchiveBoxCLIAPITestCase(TestCase):
-#     def setUp(self):
-#         self.client = TestClient(router)
+class ArchiveBoxCLIAPITestCase(TestCase):
+    def setUp(self):
+        self.client = TestClient(router)

-#     def test_add_endpoint(self):
-#         response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
+    def test_add_endpoint(self):
+        response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])

-#     def test_remove_endpoint(self):
-#         response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
+    def test_remove_endpoint(self):
+        response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])

-#     def test_update_endpoint(self):
-#         response = self.client.post("/update", json={})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
+    def test_update_endpoint(self):
+        response = self.client.post("/update", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])

-#     def test_list_all_endpoint(self):
-#         response = self.client.post("/list_all", json={})
-#         self.assertEqual(response.status_code, 200)
-#         self.assertTrue(response.json()["success"])
+    def test_list_all_endpoint(self):
+        response = self.client.post("/list_all", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])
@@ -12,13 +12,11 @@ from ninja import NinjaAPI, Swagger

 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/

-from archivebox.config import VERSION
-from archivebox.config.version import get_COMMIT_HASH
-
 from api.auth import API_AUTH_METHODS
+from ..config import VERSION, COMMIT_HASH


-COMMIT_HASH = get_COMMIT_HASH() or 'unknown'
+COMMIT_HASH = COMMIT_HASH or 'unknown'

 html_description=f'''
 <h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
@@ -54,26 +52,7 @@ class NinjaAPIWithIOCapture(NinjaAPI):

                response = super().create_temporal_response(request)

-        # Diable caching of API responses entirely
-        response['Cache-Control'] = 'no-store'
-
-        # Add debug stdout and stderr headers to response
-        response['X-ArchiveBox-Stdout'] = str(request.stdout)[200:]
-        response['X-ArchiveBox-Stderr'] = str(request.stderr)[200:]
-        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
-
-        # Add Auth Headers to response
-        api_token = getattr(request, '_api_token', None)
-        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else 'Never'
-
-        response['X-ArchiveBox-Auth-Method'] = getattr(request, '_api_auth_method', None) or 'None'
-        response['X-ArchiveBox-Auth-Expires'] = token_expiry
-        response['X-ArchiveBox-Auth-Token-Id'] = api_token.abid if api_token else 'None'
-        response['X-ArchiveBox-Auth-User-Id'] = request.user.pk if request.user.pk else 'None'
-        response['X-ArchiveBox-Auth-User-Username'] = request.user.username if request.user.pk else 'None'
-
-        # import ipdb; ipdb.set_trace()
-        # print('RESPONDING NOW', response)
+        print('RESPONDING NOW', response)

        return response

@@ -3,14 +3,12 @@ __package__ = 'archivebox.api'
 from typing import Optional

 from ninja import Router, Schema
-from django.utils import timezone
-from datetime import timedelta

 from api.models import APIToken
-from api.auth import auth_using_token, auth_using_password, get_or_create_api_token
+from api.auth import auth_using_token, auth_using_password


-router = Router(tags=['Authentication'], auth=None)
+router = Router(tags=['Authentication'])


 class PasswordAuthSchema(Schema):
@@ -27,9 +25,10 @@ def get_api_token(request, auth_data: PasswordAuthSchema):
        request=request,
    )

-    if user and user.is_superuser:
-        api_token = get_or_create_api_token(user)
-        assert api_token is not None, "Failed to create API token"
+    if user:
+        # TODO: support multiple tokens in the future, for now we just have one per user
+        api_token, created = APIToken.objects.get_or_create(created_by_id=user.pk)
+
        return api_token.__json__()
    
    return {"success": False, "errors": ["Invalid credentials"]}
@@ -12,14 +12,12 @@ from ..main import (
    list_all,
    schedule,
 )
-from archivebox.misc.util import ansi_to_html
-from archivebox.config.common import ARCHIVING_CONFIG
+from ..util import ansi_to_html
+from ..config import ONLY_NEW


-from .auth import API_AUTH_METHODS
-
 # router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=['ArchiveBox CLI Sub-Commands'], auth=API_AUTH_METHODS)
+router = Router(tags=['ArchiveBox CLI Sub-Commands'])


 # Schemas
@@ -58,7 +56,7 @@ class AddCommandSchema(Schema):
    urls: List[str]
    tag: str = ""
    depth: int = 0
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW  # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW
+    update: bool = not ONLY_NEW  # Default to the opposite of ONLY_NEW
    update_all: bool = False
    index_only: bool = False
    overwrite: bool = False
@@ -68,7 +66,7 @@ class AddCommandSchema(Schema):

 class UpdateCommandSchema(Schema):
    resume: Optional[float] = 0
-    only_new: bool = ARCHIVING_CONFIG.ONLY_NEW
+    only_new: bool = ONLY_NEW
    index_only: bool = False
    overwrite: bool = False
    after: Optional[float] = 0
@@ -85,7 +83,7 @@ class ScheduleCommandSchema(Schema):
    tag: str = ''
    depth: int = 0
    overwrite: bool = False
-    update: bool = not ARCHIVING_CONFIG.ONLY_NEW
+    update: bool = not ONLY_NEW
    clear: bool = False

 class ListCommandSchema(Schema):
@@ -94,7 +92,7 @@ class ListCommandSchema(Schema):
    status: Optional[StatusChoices] = StatusChoices.indexed
    after: Optional[float] = 0
    before: Optional[float] = 999999999999999
-    sort: str = 'bookmarked_at'
+    sort: str = 'added'
    as_json: bool = True
    as_html: bool = False
    as_csv: str | bool = 'timestamp,url'
@@ -6,19 +6,17 @@ from typing import List, Optional, Union, Any
 from datetime import datetime

 from django.db.models import Q
+from django.shortcuts import get_object_or_404
 from django.core.exceptions import ValidationError
 from django.contrib.auth import get_user_model

 from ninja import Router, Schema, FilterSchema, Field, Query
 from ninja.pagination import paginate, PaginationBase
-from ninja.errors import HttpError

 from core.models import Snapshot, ArchiveResult, Tag
-from api.models import APIToken, OutboundWebhook
+from abid_utils.abid import ABID

-from .auth import API_AUTH_METHODS
-
-router = Router(tags=['Core Models'], auth=API_AUTH_METHODS)
+router = Router(tags=['Core Models'])



@@ -58,17 +56,23 @@ class CustomPagination(PaginationBase):

 ### ArchiveResult #########################################################################

-class MinimalArchiveResultSchema(Schema):
+class ArchiveResultSchema(Schema):
    TYPE: str = 'core.models.ArchiveResult'

    id: UUID
+    old_id: int
    abid: str

-    modified_at: datetime
-    created_at: datetime
+    modified: datetime
+    created: datetime
    created_by_id: str
    created_by_username: str

+    snapshot_abid: str
+    snapshot_timestamp: str
+    snapshot_url: str
+    snapshot_tags: str
+
    extractor: str
    cmd_version: Optional[str]
    cmd: List[str]
@@ -76,9 +80,6 @@ class MinimalArchiveResultSchema(Schema):
    status: str
    output: str

-    start_ts: Optional[datetime]
-    end_ts: Optional[datetime]
-
    @staticmethod
    def resolve_created_by_id(obj):
        return str(obj.created_by_id)
@@ -88,12 +89,20 @@ class MinimalArchiveResultSchema(Schema):
        User = get_user_model()
        return User.objects.get(id=obj.created_by_id).username

+    @staticmethod
+    def resolve_pk(obj):
+        return str(obj.pk)
+
+    @staticmethod
+    def resolve_uuid(obj):
+        return str(obj.uuid)
+
    @staticmethod
    def resolve_abid(obj):
        return str(obj.ABID)

    @staticmethod
-    def resolve_created_at(obj):
+    def resolve_created(obj):
        return obj.start_ts

    @staticmethod
@@ -104,34 +113,19 @@ class MinimalArchiveResultSchema(Schema):
    def resolve_snapshot_url(obj):
        return obj.snapshot.url

-    @staticmethod
-    def resolve_snapshot_id(obj):
-        return str(obj.snapshot_id)
-    
    @staticmethod
    def resolve_snapshot_abid(obj):
        return str(obj.snapshot.ABID)

    @staticmethod
    def resolve_snapshot_tags(obj):
-        return sorted(tag.name for tag in obj.snapshot.tags.all())
-
-class ArchiveResultSchema(MinimalArchiveResultSchema):
-    TYPE: str = 'core.models.ArchiveResult'
-
-    # ... Extends MinimalArchiveResultSchema fields ...
-
-    snapshot_id: UUID
-    snapshot_abid: str
-    snapshot_timestamp: str
-    snapshot_url: str
-    snapshot_tags: List[str]
+        return obj.snapshot.tags_str()


 class ArchiveResultFilterSchema(FilterSchema):
-    id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+    id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])

-    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
    snapshot_id: Optional[str] = Field(None, q=['snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
    snapshot_url: Optional[str] = Field(None, q='snapshot__url__icontains')
    snapshot_tag: Optional[str] = Field(None, q='snapshot__tags__name__icontains')
@@ -143,9 +137,9 @@ class ArchiveResultFilterSchema(FilterSchema):
    pwd: Optional[str] = Field(None, q='pwd__icontains')
    cmd_version: Optional[str] = Field(None, q='cmd_version')

-    created_at: Optional[datetime] = Field(None, q='created_at')
-    created_at__gte: Optional[datetime] = Field(None, q='created_at__gte')
-    created_at__lt: Optional[datetime] = Field(None, q='created_at__lt')
+    created: Optional[datetime] = Field(None, q='updated')
+    created__gte: Optional[datetime] = Field(None, q='updated__gte')
+    created__lt: Optional[datetime] = Field(None, q='updated__lt')


@router.get("/archiveresults", response=List[ArchiveResultSchema], url_name="get_archiveresult")
@@ -159,8 +153,8 @@ def get_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...))

@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
 def get_archiveresult(request, archiveresult_id: str):
-    """Get a specific ArchiveResult by id or abid."""
-    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id))
+    """Get a specific ArchiveResult by pk, abid, or old_id."""
+    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id) | Q(old_id__icontains=archiveresult_id))


 # @router.post("/archiveresult", response=ArchiveResultSchema)
@@ -195,27 +189,26 @@ class SnapshotSchema(Schema):
    TYPE: str = 'core.models.Snapshot'

    id: UUID
+    old_id: UUID
    abid: str

+    modified: datetime
+    created: datetime
    created_by_id: str
    created_by_username: str
-    created_at: datetime
-    modified_at: datetime
-
-    bookmarked_at: datetime
-    downloaded_at: Optional[datetime]

    url: str
-    tags: List[str]
+    tags: str
    title: Optional[str]
    timestamp: str
    archive_path: str

-    # url_for_admin: str
-    # url_for_view: str
+    bookmarked: datetime
+    added: datetime
+    updated: Optional[datetime]

    num_archiveresults: int
-    archiveresults: List[MinimalArchiveResultSchema]
+    archiveresults: List[ArchiveResultSchema]

    @staticmethod
    def resolve_created_by_id(obj):
@@ -226,21 +219,21 @@ class SnapshotSchema(Schema):
        User = get_user_model()
        return User.objects.get(id=obj.created_by_id).username

+    @staticmethod
+    def resolve_pk(obj):
+        return str(obj.pk)
+
+    @staticmethod
+    def resolve_uuid(obj):
+        return str(obj.uuid)
+
    @staticmethod
    def resolve_abid(obj):
        return str(obj.ABID)

    @staticmethod
    def resolve_tags(obj):
-        return sorted(tag.name for tag in obj.tags.all())
-
-    # @staticmethod
-    # def resolve_url_for_admin(obj):
-    #     return f"/admin/core/snapshot/{obj.id}/change/"
-    
-    # @staticmethod
-    # def resolve_url_for_view(obj):
-    #     return f"/{obj.archive_path}"
+        return obj.tags_str()

    @staticmethod
    def resolve_num_archiveresults(obj, context):
@@ -254,27 +247,29 @@ class SnapshotSchema(Schema):


 class SnapshotFilterSchema(FilterSchema):
-    id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'timestamp__startswith'])
+    id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
+
+    old_id: Optional[str] = Field(None, q='old_id__icontains')
    abid: Optional[str] = Field(None, q='abid__icontains')

    created_by_id: str = Field(None, q='created_by_id')
    created_by_username: str = Field(None, q='created_by__username__icontains')

-    created_at__gte: datetime = Field(None, q='created_at__gte')
-    created_at__lt: datetime = Field(None, q='created_at__lt')
-    created_at: datetime = Field(None, q='created_at')
-    modified_at: datetime = Field(None, q='modified_at')
-    modified_at__gte: datetime = Field(None, q='modified_at__gte')
-    modified_at__lt: datetime = Field(None, q='modified_at__lt')
+    created__gte: datetime = Field(None, q='created__gte')
+    created__lt: datetime = Field(None, q='created__lt')
+    created: datetime = Field(None, q='created')
+    modified: datetime = Field(None, q='modified')
+    modified__gte: datetime = Field(None, q='modified__gte')
+    modified__lt: datetime = Field(None, q='modified__lt')

-    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'timestamp__startswith'])
+    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
    url: Optional[str] = Field(None, q='url')
    tag: Optional[str] = Field(None, q='tags__name')
    title: Optional[str] = Field(None, q='title__icontains')
    timestamp: Optional[str] = Field(None, q='timestamp__startswith')
    
-    bookmarked_at__gte: Optional[datetime] = Field(None, q='bookmarked_at__gte')
-    bookmarked_at__lt: Optional[datetime] = Field(None, q='bookmarked_at__lt')
+    added__gte: Optional[datetime] = Field(None, q='added__gte')
+    added__lt: Optional[datetime] = Field(None, q='added__lt')



@@ -290,16 +285,16 @@ def get_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_arch

@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
 def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
-    """Get a specific Snapshot by abid or id."""
+    """Get a specific Snapshot by abid, uuid, or pk."""
    request.with_archiveresults = with_archiveresults
    snapshot = None
    try:
-        snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
+        snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(old_id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
    except Snapshot.DoesNotExist:
        pass

    try:
-        snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id))
+        snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
    except Snapshot.DoesNotExist:
        pass

@@ -316,7 +311,7 @@ def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
 #
 # @router.put("/snapshot/{snapshot_id}", response=SnapshotSchema)
 # def update_snapshot(request, snapshot_id: str, payload: SnapshotSchema):
-#     snapshot = get_object_or_404(Snapshot, id=snapshot_id)
+#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
 #
 #     for attr, value in payload.dict().items():
 #         setattr(snapshot, attr, value)
@@ -326,7 +321,7 @@ def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
 #
 # @router.delete("/snapshot/{snapshot_id}")
 # def delete_snapshot(request, snapshot_id: str):
-#     snapshot = get_object_or_404(Snapshot, id=snapshot_id)
+#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
 #     snapshot.delete()
 #     return {"success": True}

@@ -339,10 +334,11 @@ class TagSchema(Schema):
    TYPE: str = 'core.models.Tag'

    id: UUID
+    old_id: str
    abid: str

-    modified_at: datetime
-    created_at: datetime
+    modified: datetime
+    created: datetime
    created_by_id: str
    created_by_username: str

@@ -351,6 +347,10 @@ class TagSchema(Schema):
    num_snapshots: int
    snapshots: List[SnapshotSchema]

+    @staticmethod
+    def resolve_old_id(obj):
+        return str(obj.old_id)
+
    @staticmethod
    def resolve_created_by_id(obj):
        return str(obj.created_by_id)
@@ -382,6 +382,11 @@ def get_tag(request, tag_id: str, with_snapshots: bool=True):
    request.with_snapshots = with_snapshots
    request.with_archiveresults = False
    tag = None
+    try:
+        tag = tag or Tag.objects.get(old_id__icontains=tag_id)
+    except (Tag.DoesNotExist, ValidationError, ValueError):
+        pass
+
    try:
        tag = Tag.objects.get(abid__icontains=tag_id)
    except (Tag.DoesNotExist, ValidationError):
@@ -395,70 +400,11 @@ def get_tag(request, tag_id: str, with_snapshots: bool=True):



-# class CrawlSchema(Schema):
-#     TYPE: str = 'core.models.Crawl'
-
-#     id: UUID
-#     abid: str
-
-#     modified_at: datetime
-#     created_at: datetime
-#     created_by_id: str
-#     created_by_username: str
-
-#     urls: str
-#     depth: int
-#     parser: str
-    
-#     # snapshots: List[SnapshotSchema]
-
-#     @staticmethod
-#     def resolve_created_by_id(obj):
-#         return str(obj.created_by_id)
-    
-#     @staticmethod
-#     def resolve_created_by_username(obj):
-#         User = get_user_model()
-#         return User.objects.get(id=obj.created_by_id).username
-    
-#     @staticmethod
-#     def resolve_snapshots(obj, context):
-#         if context['request'].with_snapshots:
-#             return obj.snapshot_set.all().distinct()
-#         return Snapshot.objects.none()
-
-
-# @router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
-# def get_crawl(request, crawl_id: str, with_snapshots: bool=False, with_archiveresults: bool=False):
-#     """Get a specific Crawl by id or abid."""
-#     crawl = None
-#     request.with_snapshots = with_snapshots
-#     request.with_archiveresults = with_archiveresults
-    
-#     try:
-#         crawl = Crawl.objects.get(abid__icontains=crawl_id)
-#     except Exception:
-#         pass
-
-#     try:
-#         crawl = crawl or Crawl.objects.get(id__icontains=crawl_id)
-#     except Exception:
-#         pass
-#     return crawl
-
-
-# [..., CrawlSchema]
@router.get("/any/{abid}", response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema], url_name="get_any")
 def get_any(request, abid: str):
    request.with_snapshots = False
    request.with_archiveresults = False

-    if abid.startswith(APIToken.abid_prefix):
-        raise HttpError(403, 'APIToken objects are not accessible via REST API')
-    
-    if abid.startswith(OutboundWebhook.abid_prefix):
-        raise HttpError(403, 'OutboundWebhook objects are not accessible via REST API')
-    
    response = None
    try:
        response = response or get_snapshot(request, abid)
@@ -474,13 +420,5 @@ def get_any(request, abid: str):
        response = response or get_tag(request, abid)
    except Exception:
        pass
-    
-    # try:
-    #     response = response or get_crawl(request, abid)
-    # except Exception:
-    #     pass
-
-    if not response:
-        raise HttpError(404, 'Object with given ABID not found')

    return response
@@ -0,0 +1,82 @@
+import sys
+import inspect
+from typing import List, Dict, Any, Optional
+from pathlib import Path
+
+import django
+from django.apps import AppConfig
+from django.core.checks import Tags, Warning, register
+from django.db.backends.sqlite3.base import Database as sqlite3
+
+from pydantic import (
+    Field,
+    SerializeAsAny,
+)
+
+from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
+
+from plugantic.extractors import Extractor, ExtractorName
+from plugantic.plugins import Plugin
+from plugantic.configs import ConfigSet, ConfigSectionName
+from plugantic.replayers import Replayer
+
+
+class PythonBinary(Binary):
+    name: BinName = 'python'
+
+    providers_supported: List[BinProvider] = [EnvProvider()]
+    provider_overrides: Dict[str, Any] = {
+        'env': {
+            'subdeps': \
+                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
+            'abspath': \
+                lambda: sys.executable,
+            'version': \
+                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
+        },
+    }
+
+class SqliteBinary(Binary):
+    name: BinName = 'sqlite'
+    providers_supported: List[BinProvider] = [EnvProvider()]
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'env': {
+            'abspath': \
+                lambda: inspect.getfile(sqlite3),
+            'version': \
+                lambda: sqlite3.version,
+        },
+    }
+
+class DjangoBinary(Binary):
+    name: BinName = 'django'
+
+    providers_supported: List[BinProvider] = [EnvProvider()]
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'env': {
+            'abspath': \
+                lambda: inspect.getfile(django),
+            'version': \
+                lambda: django.VERSION[:3],
+        },
+    }
+
+
+class BasicReplayer(Replayer):
+    name: str = 'basic'
+
+
+class BasePlugin(Plugin):
+    name: str = 'base'
+    configs: List[SerializeAsAny[ConfigSet]] = []
+    binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
+    extractors: List[SerializeAsAny[Extractor]] = []
+    replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()]
+
+
+PLUGINS = [BasePlugin()]
+
+
+class BaseConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'builtin_plugins.base'
@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.
@@ -0,0 +1,94 @@
+from typing import List, Optional
+from pathlib import Path
+
+from django.apps import AppConfig
+from django.core.checks import Tags, Warning, register
+
+from pydantic import (
+    Field,
+    SerializeAsAny,
+)
+
+from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider
+
+from plugantic.extractors import Extractor, ExtractorName
+from plugantic.plugins import Plugin
+from plugantic.configs import ConfigSet, ConfigSectionName
+
+
+###################### Config ##########################
+
+class SinglefileToggleConfig(ConfigSet):
+    section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
+
+    SAVE_SINGLEFILE: bool = True
+
+
+class SinglefileDependencyConfig(ConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    SINGLEFILE_BINARY: str = Field(default='wget')
+    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
+    SINGLEFILE_EXTRA_ARGS: List[str] = []
+    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
+
+class SinglefileOptionsConfig(ConfigSet):
+    section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
+
+    # loaded from shared config
+    SINGLEFILE_USER_AGENT: str = Field(default='', alias='USER_AGENT')
+    SINGLEFILE_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
+    SINGLEFILE_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
+    SINGLEFILE_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
+    SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
+
+
+
+DEFAULT_CONFIG = {
+    'CHECK_SSL_VALIDITY': False,
+    'SAVE_SINGLEFILE': True,
+    'TIMEOUT': 120,
+}
+
+PLUGIN_CONFIG = [
+    SinglefileToggleConfig(**DEFAULT_CONFIG),
+    SinglefileDependencyConfig(**DEFAULT_CONFIG),
+    SinglefileOptionsConfig(**DEFAULT_CONFIG),
+]
+
+###################### Binaries ############################
+
+class SinglefileBinary(Binary):
+    name: BinName = 'single-file'
+    providers_supported: List[BinProvider] = [EnvProvider(), NpmProvider()]
+
+
+###################### Extractors ##########################
+
+class SinglefileExtractor(Extractor):
+    name: ExtractorName = 'singlefile'
+    binary: Binary = SinglefileBinary()
+
+    def get_output_path(self, snapshot) -> Path:
+        return Path(snapshot.link_dir) / 'singlefile.html'
+
+
+###################### Plugins #############################
+
+
+class SinglefilePlugin(Plugin):
+    name: str = 'singlefile'
+    configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG]
+    binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()]
+    extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()]
+
+PLUGINS = [SinglefilePlugin()]
+
+###################### Django Apps #########################
+
+class SinglefileConfig(AppConfig):
+    name = 'builtin_plugins.singlefile'
+    verbose_name = 'SingleFile'
+
+    def ready(self):
+        print('Loaded singlefile plugin')
@@ -0,0 +1,66 @@
+name: singlefile
+plugin_version: '0.0.1'
+plugin_spec: '0.0.1'
+
+binaries:
+    singlefile:
+        providers:
+            - env
+            - npm
+
+commands:
+    - singlefile.exec
+    - singlefile.extract
+    - singlefile.should_extract
+    - singlefile.get_output_path
+
+extractors:
+    singlefile:
+        binary: singlefile
+        test: singlefile.should_extract
+        extract: singlefile.extract
+        output_files:
+            - singlefile.html
+
+configs:
+    ARCHIVE_METHOD_TOGGLES:
+        SAVE_SINGLEFILE:
+            type: bool
+            default: true
+
+    DEPENDENCY_CONFIG:
+        SINGLEFILE_BINARY:
+            type: str
+            default: wget
+        SINGLEFILE_ARGS:
+            type: Optional[List[str]]
+            default: null
+        SINGLEFILE_EXTRA_ARGS:
+            type: List[str]
+            default: []
+        SINGLEFILE_DEFAULT_ARGS:
+            type: List[str]
+            default: 
+            - "--timeout={TIMEOUT-10}"
+
+    ARCHIVE_METHOD_OPTIONS:
+        SINGLEFILE_USER_AGENT:
+            type: str
+            default: ""
+            alias: USER_AGENT
+        SINGLEFILE_TIMEOUT:
+            type: int
+            default: 60
+            alias: TIMEOUT
+        SINGLEFILE_CHECK_SSL_VALIDITY:
+            type: bool
+            default: true
+            alias: CHECK_SSL_VALIDITY
+        SINGLEFILE_RESTRICT_FILE_NAMES:
+            type: str
+            default: windows
+            alias: RESTRICT_FILE_NAMES
+        SINGLEFILE_COOKIES_FILE:
+            type: Optional[Path]
+            default: null
+            alias: COOKIES_FILE
@@ -5,71 +5,26 @@ import os
 import sys
 import argparse
 import threading
-
 from time import sleep
-from collections.abc import Mapping

-from rich import print
-
-from typing import Optional, List, IO, Union, Iterable
+from typing import Optional, Dict, List, IO, Union, Iterable
 from pathlib import Path

+from ..config import OUTPUT_DIR, check_data_folder, check_migrations, stderr
+
 from importlib import import_module

 BUILTIN_LIST = list

 CLI_DIR = Path(__file__).resolve().parent

-# rewrite setup -> install for backwards compatibility
-if len(sys.argv) > 1 and sys.argv[1] == 'setup':
-    from rich import print
-    print(':warning: [bold red]DEPRECATED[/bold red] `archivebox setup` is deprecated, use `archivebox install` instead')
-    sys.argv[1] = 'install'
+# these common commands will appear sorted before any others for ease-of-use
+meta_cmds = ('help', 'version')                               # dont require valid data folder at all
+main_cmds = ('init', 'config', 'setup')                       # dont require existing db present
+archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
+fake_db = ("oneshot",)                                        # use fake in-memory db

-if '--debug' in sys.argv:
-    os.environ['DEBUG'] = 'True'
-    sys.argv.remove('--debug')
-
-
-# def list_subcommands() -> Dict[str, str]:
-#     """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
-#     COMMANDS = []
-#     for filename in os.listdir(CLI_DIR):
-#         if is_cli_module(filename):
-#             subcommand = filename.replace('archivebox_', '').replace('.py', '')
-#             module = import_module('.archivebox_{}'.format(subcommand), __package__)
-#             assert is_valid_cli_module(module, subcommand)
-#             COMMANDS.append((subcommand, module.main.__doc__))
-#             globals()[subcommand] = module.main
-#     display_order = lambda cmd: (
-#         display_first.index(cmd[0])
-#         if cmd[0] in display_first else
-#         100 + len(cmd[0])
-#     )
-#     return dict(sorted(COMMANDS, key=display_order))
-
-# just define it statically, it's much faster:
-SUBCOMMAND_MODULES = {
-    'help': 'archivebox_help',
-    'version': 'archivebox_version' ,
-    
-    'init': 'archivebox_init',
-    'install': 'archivebox_install',
-    ##############################################
-    'config': 'archivebox_config',
-    'add': 'archivebox_add',
-    'remove': 'archivebox_remove',
-    'update': 'archivebox_update',
-    'list': 'archivebox_list',
-    'status': 'archivebox_status',
-    
-    'schedule': 'archivebox_schedule',
-    'server': 'archivebox_server',
-    'shell': 'archivebox_shell',
-    'manage': 'archivebox_manage',
-
-    # 'oneshot': 'archivebox_oneshot',
-}
+display_first = (*meta_cmds, *main_cmds, *archive_cmds)

 # every imported command module must have these properties in order to be valid
 required_attrs = ('__package__', '__command__', 'main')
@@ -81,40 +36,8 @@ is_valid_cli_module = lambda module, subcommand: (
    and module.__command__.split(' ')[-1] == subcommand
 )

-class LazySubcommands(Mapping):
-    def keys(self):
-        return SUBCOMMAND_MODULES.keys()
-    
-    def values(self):
-        return [self[key] for key in self.keys()]
-    
-    def items(self):
-        return [(key, self[key]) for key in self.keys()]
-    
-    def __getitem__(self, key):
-        module = import_module(f'.{SUBCOMMAND_MODULES[key]}', __package__)
-        assert is_valid_cli_module(module, key)
-        return module.main
-    
-    def __iter__(self):
-        return iter(SUBCOMMAND_MODULES.keys())
-    
-    def __len__(self):
-        return len(SUBCOMMAND_MODULES)

-CLI_SUBCOMMANDS = LazySubcommands()
-
-
-# these common commands will appear sorted before any others for ease-of-use
-meta_cmds = ('help', 'version')                               # dont require valid data folder at all
-setup_cmds = ('init', 'setup', 'install')                      # require valid data folder, but dont require DB present in it yet
-archive_cmds = ('add', 'remove', 'update', 'list', 'status', 'schedule', 'server', 'shell', 'manage')  # require valid data folder + existing db present
-fake_db = ("oneshot",)                                        # use fake in-memory db
-
-display_first = (*meta_cmds, *setup_cmds, *archive_cmds)
-
-
-IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread', 'Scheduler')  # threads we dont have to wait for before exiting
+IGNORED_BG_THREADS = ('MainThread', 'ThreadPoolExecutor', 'IPythonHistorySavingThread')  # threads we dont have to wait for before exiting


 def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: Iterable[str]=IGNORED_BG_THREADS, timeout: int=60) -> int:
@@ -138,45 +61,56 @@ def wait_for_bg_threads_to_exit(thread_names: Iterable[str]=(), ignore_names: It
        if blocking_threads:
            sleep(1)
            if tries == 5:                            # only show stderr message if we need to wait more than 5s
-                print(
+                stderr(
                    f'[…] Waiting up to {timeout}s for background jobs (e.g. webhooks) to finish...',
                    threads_summary,
-                    file=sys.stderr,
                )
        else:
            return tries

-    raise Exception(f'Background threads failed to exit after {tries}s: {threads_summary}')
+    raise Exception('Background threads failed to exit after {tries}s: {threads_summary}')


+def list_subcommands() -> Dict[str, str]:
+    """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
+
+    COMMANDS = []
+    for filename in os.listdir(CLI_DIR):
+        if is_cli_module(filename):
+            subcommand = filename.replace('archivebox_', '').replace('.py', '')
+            module = import_module('.archivebox_{}'.format(subcommand), __package__)
+            assert is_valid_cli_module(module, subcommand)
+            COMMANDS.append((subcommand, module.main.__doc__))
+            globals()[subcommand] = module.main
+
+    display_order = lambda cmd: (
+        display_first.index(cmd[0])
+        if cmd[0] in display_first else
+        100 + len(cmd[0])
+    )
+
+    return dict(sorted(COMMANDS, key=display_order))
+

 def run_subcommand(subcommand: str,
-                   subcommand_args: List[str] | None = None,
+                   subcommand_args: List[str]=None,
                   stdin: Optional[IO]=None,
                   pwd: Union[Path, str, None]=None) -> None:
    """Run a given ArchiveBox subcommand with the given list of args"""

    subcommand_args = subcommand_args or []

-    from archivebox.misc.checks import check_migrations
-    from archivebox.config.django import setup_django
-    
-    # print('DATA_DIR is', DATA_DIR)
-    # print('pwd is', os.getcwd())    
+    if subcommand not in meta_cmds:
+        from ..config import setup_django

-    cmd_requires_db = (subcommand in archive_cmds)
-    init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
+        cmd_requires_db = subcommand in archive_cmds
+        init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args

-    check_db = cmd_requires_db and not init_pending
+        if cmd_requires_db:
+            check_data_folder(pwd)

-    setup_django(in_memory_db=subcommand in fake_db, check_db=check_db)
+        setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)

-    for ignore_pattern in ('help', '-h', '--help', 'version', '--version'):
-        if ignore_pattern in sys.argv[:4]:
-            cmd_requires_db = False
-            break
-    
-    if subcommand in archive_cmds:
        if cmd_requires_db:
            check_migrations()

@@ -187,28 +121,17 @@ def run_subcommand(subcommand: str,
    wait_for_bg_threads_to_exit(timeout=60)


-
-
+SUBCOMMANDS = list_subcommands()

 class NotProvided:
-    def __len__(self):
-        return 0
-    def __bool__(self):
-        return False
-    def __repr__(self):
-        return '<not provided>'
-
-Omitted = Union[None, NotProvided]
-
-OMITTED = NotProvided()
+    pass


-def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: str | None=None) -> None:
-    # print('STARTING CLI MAIN ENTRYPOINT')
-    
-    args = sys.argv[1:] if args is OMITTED else args
-    stdin = sys.stdin if stdin is OMITTED else stdin
+def main(args: Optional[List[str]]=NotProvided, stdin: Optional[IO]=NotProvided, pwd: Optional[str]=None) -> None:
+    args = sys.argv[1:] if args is NotProvided else args
+    stdin = sys.stdin if stdin is NotProvided else stdin

+    subcommands = list_subcommands()
    parser = argparse.ArgumentParser(
        prog=__command__,
        description='ArchiveBox: The self-hosted internet archive',
@@ -218,19 +141,19 @@ def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: st
    group.add_argument(
        '--help', '-h',
        action='store_true',
-        help=CLI_SUBCOMMANDS['help'].__doc__,
+        help=subcommands['help'],
    )
    group.add_argument(
        '--version',
        action='store_true',
-        help=CLI_SUBCOMMANDS['version'].__doc__,
+        help=subcommands['version'],
    )
    group.add_argument(
        "subcommand",
        type=str,
        help= "The name of the subcommand to run",
        nargs='?',
-        choices=CLI_SUBCOMMANDS.keys(),
+        choices=subcommands.keys(),
        default=None,
    )
    parser.add_argument(
@@ -245,20 +168,29 @@ def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: st
    elif command.help or command.subcommand is None:
        command.subcommand = 'help'

-    if command.subcommand not in ('version',):
+    if command.subcommand not in ('help', 'version', 'status'):
        from ..logging_util import log_cli_command

        log_cli_command(
            subcommand=command.subcommand,
            subcommand_args=command.subcommand_args,
-            stdin=stdin or None,
+            stdin=stdin,
+            pwd=pwd or OUTPUT_DIR
        )

-    try:
-        run_subcommand(
-            subcommand=command.subcommand,
-            subcommand_args=command.subcommand_args,
-            stdin=stdin or None,
-        )
-    except KeyboardInterrupt:
-        print('\n\n[red][X] Got CTRL+C. Exiting...[/red]')
+    run_subcommand(
+        subcommand=command.subcommand,
+        subcommand_args=command.subcommand_args,
+        stdin=stdin,
+        pwd=pwd or OUTPUT_DIR,
+    )
+
+
+__all__ = (
+    'SUBCOMMANDS',
+    'list_subcommands',
+    'run_subcommand',
+    *SUBCOMMANDS.keys(),
+)
+
+
@@ -8,12 +8,10 @@ import argparse

 from typing import List, Optional, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from archivebox.config.common import ARCHIVING_CONFIG
-
 from ..main import add
+from ..util import docstring
 from ..parsers import PARSERS
+from ..config import OUTPUT_DIR, ONLY_NEW
 from ..logging_util import SmartFormatter, accept_stdin, stderr


@@ -34,7 +32,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
    parser.add_argument(
        '--update', #'-u',
        action='store_true',
-        default=not ARCHIVING_CONFIG.ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
+        default=not ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
        help="Also retry previously skipped/failed links when adding new links",
    )
    parser.add_argument(
@@ -119,7 +117,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
        init=command.init,
        extractors=command.extract,
        parser=command.parser,
-        out_dir=pwd or DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
    )


@@ -5,13 +5,12 @@ __command__ = 'archivebox config'

 import sys
 import argparse
-from pathlib import Path

 from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
 from ..main import config
+from ..util import docstring
+from ..config import OUTPUT_DIR
 from ..logging_util import SmartFormatter, accept_stdin


@@ -24,11 +23,6 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
        formatter_class=SmartFormatter,
    )
    group = parser.add_mutually_exclusive_group()
-    parser.add_argument(
-        '--search',
-        action='store_true',
-        help="Search config KEYs, VALUEs, and ALIASES for the given term",
-    )
    group.add_argument(
        '--get', #'-g',
        action='store_true',
@@ -59,11 +53,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
    config(
        config_options_str=config_options_str,
        config_options=command.config_options,
-        search=command.search,
        get=command.get,
        set=command.set,
        reset=command.reset,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
    )


@@ -5,12 +5,12 @@ __command__ = 'archivebox help'

 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
 from ..main import help
+from ..util import docstring
+from ..config import OUTPUT_DIR
 from ..logging_util import SmartFormatter, reject_stdin


@@ -25,7 +25,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
    parser.parse_args(args or ())
    reject_stdin(__command__, stdin)
    
-    help(out_dir=Path(pwd) if pwd else DATA_DIR)
+    help(out_dir=pwd or OUTPUT_DIR)


 if __name__ == '__main__':
@@ -8,10 +8,10 @@ import argparse

 from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
 from ..main import init
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin


@docstring(init.__doc__)
@@ -32,15 +32,10 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
        action='store_true',
        help='Run any updates or migrations without rechecking all snapshot dirs',
    )
-    parser.add_argument(
-        '--install', #'-s',
-        action='store_true',
-        help='Automatically install dependencies and extras used for archiving',
-    )
    parser.add_argument(
        '--setup', #'-s',
        action='store_true',
-        help='DEPRECATED: equivalent to --install',
+        help='Automatically install dependencies and extras used for archiving',
    )
    command = parser.parse_args(args or ())
    reject_stdin(__command__, stdin)
@@ -48,8 +43,8 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
    init(
        force=command.force,
        quick=command.quick,
-        install=command.install or command.setup,
-        out_dir=pwd or DATA_DIR,
+        setup=command.setup,
+        out_dir=pwd or OUTPUT_DIR,
    )
    

@@ -1,56 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = 'archivebox.cli'
-__command__ = 'archivebox install'
-
-import sys
-import argparse
-from pathlib import Path
-from typing import Optional, List, IO
-
-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
-from ..main import install
-
-
-@docstring(install.__doc__)
-def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    parser = argparse.ArgumentParser(
-        prog=__command__,
-        description=install.__doc__,
-        add_help=True,
-        formatter_class=SmartFormatter,
-    )
-    parser.add_argument(
-        '--binproviders', '-p',
-        type=str,
-        help='Select binproviders to use DEFAULT=env,apt,brew,sys_pip,venv_pip,lib_pip,pipx,sys_npm,lib_npm,puppeteer,playwright (all)',
-        default=None,
-    )
-    parser.add_argument(
-        '--binaries', '-b',
-        type=str,
-        help='Select binaries to install DEFAULT=curl,wget,git,yt-dlp,chrome,single-file,readability-extractor,postlight-parser,... (all)',
-        default=None,
-    )
-    parser.add_argument(
-        '--dry-run', '-d',
-        action='store_true',
-        help='Show what would be installed without actually installing anything',
-        default=False,
-    )
-    command = parser.parse_args(args or ())   # noqa
-    reject_stdin(__command__, stdin)
-
-    install(
-        # force=command.force,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-        binaries=command.binaries.split(',') if command.binaries else None,
-        binproviders=command.binproviders.split(',') if command.binproviders else None,
-        dry_run=command.dry_run,
-    )
-    
-
-if __name__ == '__main__':
-    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -5,12 +5,12 @@ __command__ = 'archivebox list'

 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
 from ..main import list_all
+from ..util import docstring
+from ..config import OUTPUT_DIR
 from ..index import (
    LINK_FILTERS,
    get_indexed_folders,
@@ -131,7 +131,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
        json=command.json,
        html=command.html,
        with_headers=command.with_headers,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
    )
    raise SystemExit(not matching_folders)

@@ -4,19 +4,19 @@ __package__ = 'archivebox.cli'
 __command__ = 'archivebox manage'

 import sys
-from pathlib import Path
+
 from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
 from ..main import manage
+from ..util import docstring
+from ..config import OUTPUT_DIR


@docstring(manage.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
    manage(
        args=args,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
    )


@@ -9,10 +9,10 @@ import argparse
 from pathlib import Path
 from typing import List, Optional, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, accept_stdin, stderr
 from ..main import oneshot
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, accept_stdin, stderr


@docstring(oneshot.__doc__)
@@ -46,7 +46,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
    parser.add_argument(
        '--out-dir',
        type=str,
-        default=DATA_DIR,
+        default=OUTPUT_DIR,
        help= "Path to save the single archive folder to, e.g. ./example.com_archive"
    )
    command = parser.parse_args(args or ())
@@ -5,13 +5,13 @@ __command__ = 'archivebox remove'

 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, accept_stdin
 from ..main import remove
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, accept_stdin


@docstring(remove.__doc__)
@@ -74,7 +74,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
        after=command.after,
        yes=command.yes,
        delete=command.delete,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
    )
    

@@ -5,13 +5,13 @@ __command__ = 'archivebox schedule'

 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
 from ..main import schedule
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin


@docstring(schedule.__doc__)
@@ -108,7 +108,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
        overwrite=command.overwrite,
        update=command.update,
        import_path=command.import_path,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
    )


@@ -5,14 +5,13 @@ __command__ = 'archivebox server'

 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from archivebox.config.common import SERVER_CONFIG
-from ..logging_util import SmartFormatter, reject_stdin
 from ..main import server
+from ..util import docstring
+from ..config import OUTPUT_DIR, BIND_ADDR
+from ..logging_util import SmartFormatter, reject_stdin

@docstring(server.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
@@ -26,7 +25,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
        'runserver_args',
        nargs='*',
        type=str,
-        default=[SERVER_CONFIG.BIND_ADDR],
+        default=[BIND_ADDR],
        help='Arguments to pass to Django runserver'
    )
    parser.add_argument(
@@ -59,11 +58,6 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
        action='store_true',
        help='Run archivebox manage createsuperuser before starting the server',
    )
-    parser.add_argument(
-        '--daemonize',
-        action='store_true',
-        help='Run the server in the background as a daemon',
-    )
    command = parser.parse_args(args or ())
    reject_stdin(__command__, stdin)
    
@@ -74,8 +68,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
        init=command.init,
        quick_init=command.quick_init,
        createsuperuser=command.createsuperuser,
-        daemonize=command.daemonize,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
    )


@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox setup'
+
+import sys
+import argparse
+
+from typing import Optional, List, IO
+
+from ..main import setup
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin
+
+
+@docstring(setup.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=setup.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    # parser.add_argument(
+    #     '--force', # '-f',
+    #     action='store_true',
+    #     help='Overwrite any existing packages that conflict with the ones ArchiveBox is trying to install',
+    # )
+    command = parser.parse_args(args or ())   # noqa
+    reject_stdin(__command__, stdin)
+
+    setup(
+        # force=command.force,
+        out_dir=pwd or OUTPUT_DIR,
+    )
+    
+
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -5,13 +5,13 @@ __command__ = 'archivebox shell'

 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
 from ..main import shell
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin


@docstring(shell.__doc__)
@@ -26,7 +26,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
    reject_stdin(__command__, stdin)
    
    shell(
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
    )
    

@@ -5,13 +5,13 @@ __command__ = 'archivebox status'

 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
-from ..logging_util import SmartFormatter, reject_stdin
 from ..main import status
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin


@docstring(status.__doc__)
@@ -25,7 +25,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
    parser.parse_args(args or ())
    reject_stdin(__command__, stdin)

-    status(out_dir=Path(pwd) if pwd else DATA_DIR)
+    status(out_dir=pwd or OUTPUT_DIR)


 if __name__ == '__main__':
@@ -5,11 +5,12 @@ __command__ = 'archivebox update'

 import sys
 import argparse
-from pathlib import Path
+
 from typing import List, Optional, IO

-from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR
+from ..main import update
+from ..util import docstring
+from ..config import OUTPUT_DIR
 from ..index import (
    LINK_FILTERS,
    get_indexed_folders,
@@ -24,7 +25,6 @@ from ..index import (
    get_unrecognized_folders,
 )
 from ..logging_util import SmartFormatter, accept_stdin
-from ..main import update


@docstring(update.__doc__)
@@ -127,7 +127,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
        status=command.status,
        after=command.after,
        before=command.before,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
+        out_dir=pwd or OUTPUT_DIR,
        extractors=command.extract,
    )
    
@@ -5,20 +5,20 @@ __command__ = 'archivebox version'

 import sys
 import argparse
-from pathlib import Path
+
 from typing import Optional, List, IO

-# from archivebox.misc.util import docstring
-from archivebox.config import DATA_DIR, VERSION
+from ..main import version
+from ..util import docstring
+from ..config import OUTPUT_DIR
 from ..logging_util import SmartFormatter, reject_stdin


-# @docstring(version.__doc__)
+@docstring(version.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
-    """Print the ArchiveBox version and dependency information"""
    parser = argparse.ArgumentParser(
        prog=__command__,
-        description="Print the ArchiveBox version and dependency information",   # version.__doc__,
+        description=version.__doc__,
        add_help=True,
        formatter_class=SmartFormatter,
    )
@@ -27,33 +27,12 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
        action='store_true',
        help='Only print ArchiveBox version number and nothing else.',
    )
-    parser.add_argument(
-        '--binproviders', '-p',
-        type=str,
-        help='Select binproviders to detect DEFAULT=env,apt,brew,sys_pip,venv_pip,lib_pip,pipx,sys_npm,lib_npm,puppeteer,playwright (all)',
-        default=None,
-    )
-    parser.add_argument(
-        '--binaries', '-b',
-        type=str,
-        help='Select binaries to detect DEFAULT=curl,wget,git,yt-dlp,chrome,single-file,readability-extractor,postlight-parser,... (all)',
-        default=None,
-    )
    command = parser.parse_args(args or ())
    reject_stdin(__command__, stdin)
    
-    # for speed reasons, check if quiet flag was set and just return simple version immediately if so
-    if command.quiet:
-        print(VERSION)
-        return
-    
-    # otherwise do big expensive import to get the full version
-    from ..main import version
    version(
        quiet=command.quiet,
-        out_dir=Path(pwd) if pwd else DATA_DIR,
-        binproviders=command.binproviders.split(',') if command.binproviders else None,
-        binaries=command.binaries.split(',') if command.binaries else None,
+        out_dir=pwd or OUTPUT_DIR,
    )


@@ -15,7 +15,7 @@ TEST_CONFIG = {
    'USE_COLOR': 'False',
    'SHOW_PROGRESS': 'False',

-    'DATA_DIR': 'data.tests',
+    'OUTPUT_DIR': 'data.tests',
    
    'SAVE_ARCHIVE_DOT_ORG': 'False',
    'SAVE_TITLE': 'False',
@@ -27,12 +27,12 @@ TEST_CONFIG = {
    'USE_YOUTUBEDL': 'False',
 }

-DATA_DIR = 'data.tests'
+OUTPUT_DIR = 'data.tests'
 os.environ.update(TEST_CONFIG)

 from ..main import init
 from ..index import load_main_index
-from archivebox.config.constants import (
+from ..config import (
    SQL_INDEX_FILENAME,
    JSON_INDEX_FILENAME,
    HTML_INDEX_FILENAME,
@@ -101,22 +101,22 @@ def output_hidden(show_failing=True):

 class TestInit(unittest.TestCase):
    def setUp(self):
-        os.makedirs(DATA_DIR, exist_ok=True)
+        os.makedirs(OUTPUT_DIR, exist_ok=True)

    def tearDown(self):
-        shutil.rmtree(DATA_DIR, ignore_errors=True)
+        shutil.rmtree(OUTPUT_DIR, ignore_errors=True)

    def test_basic_init(self):
        with output_hidden():
            archivebox_init.main([])

-        assert (Path(DATA_DIR) / SQL_INDEX_FILENAME).exists()
-        assert (Path(DATA_DIR) / JSON_INDEX_FILENAME).exists()
-        assert (Path(DATA_DIR) / HTML_INDEX_FILENAME).exists()
-        assert len(load_main_index(out_dir=DATA_DIR)) == 0
+        assert (Path(OUTPUT_DIR) / SQL_INDEX_FILENAME).exists()
+        assert (Path(OUTPUT_DIR) / JSON_INDEX_FILENAME).exists()
+        assert (Path(OUTPUT_DIR) / HTML_INDEX_FILENAME).exists()
+        assert len(load_main_index(out_dir=OUTPUT_DIR)) == 0

    def test_conflicting_init(self):
-        with open(Path(DATA_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
+        with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
            f.write('test')

        try:
@@ -126,11 +126,11 @@ class TestInit(unittest.TestCase):
        except SystemExit:
            pass

-        assert not (Path(DATA_DIR) / SQL_INDEX_FILENAME).exists()
-        assert not (Path(DATA_DIR) / JSON_INDEX_FILENAME).exists()
-        assert not (Path(DATA_DIR) / HTML_INDEX_FILENAME).exists()
+        assert not (Path(OUTPUT_DIR) / SQL_INDEX_FILENAME).exists()
+        assert not (Path(OUTPUT_DIR) / JSON_INDEX_FILENAME).exists()
+        assert not (Path(OUTPUT_DIR) / HTML_INDEX_FILENAME).exists()
        try:
-            load_main_index(out_dir=DATA_DIR)
+            load_main_index(out_dir=OUTPUT_DIR)
            assert False, 'load_main_index should raise an exception when no index is present'
        except Exception:
            pass
@@ -138,36 +138,36 @@ class TestInit(unittest.TestCase):
    def test_no_dirty_state(self):
        with output_hidden():
            init()
-        shutil.rmtree(DATA_DIR, ignore_errors=True)
+        shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
        with output_hidden():
            init()


 class TestAdd(unittest.TestCase):
    def setUp(self):
-        os.makedirs(DATA_DIR, exist_ok=True)
+        os.makedirs(OUTPUT_DIR, exist_ok=True)
        with output_hidden():
            init()

    def tearDown(self):
-        shutil.rmtree(DATA_DIR, ignore_errors=True)
+        shutil.rmtree(OUTPUT_DIR, ignore_errors=True)

    def test_add_arg_url(self):
        with output_hidden():
            archivebox_add.main(['https://getpocket.com/users/nikisweeting/feed/all'])

-        all_links = load_main_index(out_dir=DATA_DIR)
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
        assert len(all_links) == 30

    def test_add_arg_file(self):
-        test_file = Path(DATA_DIR) / 'test.txt'
+        test_file = Path(OUTPUT_DIR) / 'test.txt'
        with open(test_file, 'w+', encoding='utf') as f:
            f.write(test_urls)

        with output_hidden():
            archivebox_add.main([test_file])

-        all_links = load_main_index(out_dir=DATA_DIR)
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
        assert len(all_links) == 12
        os.remove(test_file)

@@ -175,40 +175,40 @@ class TestAdd(unittest.TestCase):
        with output_hidden():
            archivebox_add.main([], stdin=test_urls)

-        all_links = load_main_index(out_dir=DATA_DIR)
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
        assert len(all_links) == 12


 class TestRemove(unittest.TestCase):
    def setUp(self):
-        os.makedirs(DATA_DIR, exist_ok=True)
+        os.makedirs(OUTPUT_DIR, exist_ok=True)
        with output_hidden():
            init()
            archivebox_add.main([], stdin=test_urls)

    # def tearDown(self):
-        # shutil.rmtree(DATA_DIR, ignore_errors=True)
+        # shutil.rmtree(OUTPUT_DIR, ignore_errors=True)


    def test_remove_exact(self):
        with output_hidden():
            archivebox_remove.main(['--yes', '--delete', 'https://example5.com/'])

-        all_links = load_main_index(out_dir=DATA_DIR)
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
        assert len(all_links) == 11

    def test_remove_regex(self):
        with output_hidden():
            archivebox_remove.main(['--yes', '--delete', '--filter-type=regex', r'http(s)?:\/\/(.+\.)?(example\d\.com)'])

-        all_links = load_main_index(out_dir=DATA_DIR)
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
        assert len(all_links) == 4

    def test_remove_domain(self):
        with output_hidden():
            archivebox_remove.main(['--yes', '--delete', '--filter-type=domain', 'example5.com', 'example6.com'])

-        all_links = load_main_index(out_dir=DATA_DIR)
+        all_links = load_main_index(out_dir=OUTPUT_DIR)
        assert len(all_links) == 10

    def test_remove_none(self):
@@ -1,36 +0,0 @@
-__package__ = 'config'
-__order__ = 200
-
-from .paths import (
-    PACKAGE_DIR,                                    # noqa
-    DATA_DIR,                                       # noqa
-    ARCHIVE_DIR,                                    # noqa
-)
-from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR      # noqa
-from .version import VERSION                        # noqa
-
-# import abx
-
-# @abx.hookimpl
-# def get_CONFIG():
-#     from .common import (
-#         SHELL_CONFIG,
-#         STORAGE_CONFIG,
-#         GENERAL_CONFIG,
-#         SERVER_CONFIG,
-#         ARCHIVING_CONFIG,
-#         SEARCH_BACKEND_CONFIG,
-#     )
-#     return {
-#         'SHELL_CONFIG': SHELL_CONFIG,
-#         'STORAGE_CONFIG': STORAGE_CONFIG,
-#         'GENERAL_CONFIG': GENERAL_CONFIG,
-#         'SERVER_CONFIG': SERVER_CONFIG,
-#         'ARCHIVING_CONFIG': ARCHIVING_CONFIG,
-#         'SEARCHBACKEND_CONFIG': SEARCH_BACKEND_CONFIG,
-#     }
-
-# @abx.hookimpl
-# def ready():
-#     for config in get_CONFIG().values():
-#         config.validate()
@@ -1,245 +0,0 @@
-__package__ = 'archivebox.config'
-
-import os
-import json
-from typing import Any, Optional, Type, Tuple, Dict
-
-from pathlib import Path
-from configparser import ConfigParser
-
-from benedict import benedict
-
-import archivebox
-
-from archivebox.config.constants import CONSTANTS
-
-from archivebox.misc.logging import stderr
-
-
-def get_real_name(key: str) -> str:
-    """get the up-to-date canonical name for a given old alias or current key"""
-    CONFIGS = archivebox.pm.hook.get_CONFIGS()
-    
-    for section in CONFIGS.values():
-        try:
-            return section.aliases[key]
-        except KeyError:
-            pass
-    return key
-
-
-def load_config_val(key: str,
-                    default: Any=None,
-                    type: Optional[Type]=None,
-                    aliases: Optional[Tuple[str, ...]]=None,
-                    config: Optional[benedict]=None,
-                    env_vars: Optional[os._Environ]=None,
-                    config_file_vars: Optional[Dict[str, str]]=None) -> Any:
-    """parse bool, int, and str key=value pairs from env"""
-
-    assert isinstance(config, dict)
-
-    is_read_only = type is None
-    if is_read_only:
-        if callable(default):
-            return default(config)
-        return default
-
-    # get value from environment variables or config files
-    config_keys_to_check = (key, *(aliases or ()))
-    val = None
-    for key in config_keys_to_check:
-        if env_vars:
-            val = env_vars.get(key)
-            if val:
-                break
-
-        if config_file_vars:
-            val = config_file_vars.get(key)
-            if val:
-                break
-
-    is_unset = val is None
-    if is_unset:
-        if callable(default):
-            return default(config)
-        return default
-
-    # calculate value based on expected type
-    BOOL_TRUEIES = ('true', 'yes', '1')
-    BOOL_FALSEIES = ('false', 'no', '0')
-
-    if type is bool:
-        if val.lower() in BOOL_TRUEIES:
-            return True
-        elif val.lower() in BOOL_FALSEIES:
-            return False
-        else:
-            raise ValueError(f'Invalid configuration option {key}={val} (expected a boolean: True/False)')
-
-    elif type is str:
-        if val.lower() in (*BOOL_TRUEIES, *BOOL_FALSEIES):
-            raise ValueError(f'Invalid configuration option {key}={val} (expected a string, but value looks like a boolean)')
-        return val.strip()
-
-    elif type is int:
-        if not val.strip().isdigit():
-            raise ValueError(f'Invalid configuration option {key}={val} (expected an integer)')
-        return int(val.strip())
-
-    elif type is list or type is dict:
-        return json.loads(val)
-    
-    elif type is Path:
-        return Path(val)
-
-    raise Exception('Config values can only be str, bool, int, or json')
-
-
-def load_config_file() -> Optional[benedict]:
-    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
-
-    config_path = CONSTANTS.CONFIG_FILE
-    if os.access(config_path, os.R_OK):
-        config_file = ConfigParser()
-        config_file.optionxform = str
-        config_file.read(config_path)
-        # flatten into one namespace
-        config_file_vars = benedict({
-            key.upper(): val
-            for section, options in config_file.items()
-                for key, val in options.items()
-        })
-        # print('[i] Loaded config file', os.path.abspath(config_path))
-        # print(config_file_vars)
-        return config_file_vars
-    return None
-
-
-def section_for_key(key: str) -> Any:
-    for config_section in archivebox.pm.hook.get_CONFIGS().values():
-        if hasattr(config_section, key):
-            return config_section
-    raise ValueError(f'No config section found for key: {key}')
-
-
-def write_config_file(config: Dict[str, str]) -> benedict:
-    """load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
-
-    from archivebox.misc.system import atomic_write
-
-    CONFIG_HEADER = (
-    """# This is the config file for your ArchiveBox collection.
-    #
-    # You can add options here manually in INI format, or automatically by running:
-    #    archivebox config --set KEY=VALUE
-    #
-    # If you modify this file manually, make sure to update your archive after by running:
-    #    archivebox init
-    #
-    # A list of all possible config with documentation and examples can be found here:
-    #    https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration
-
-    """)
-
-    config_path = CONSTANTS.CONFIG_FILE
-
-    if not os.access(config_path, os.F_OK):
-        atomic_write(config_path, CONFIG_HEADER)
-
-    config_file = ConfigParser()
-    config_file.optionxform = str
-    config_file.read(config_path)
-
-    with open(config_path, 'r', encoding='utf-8') as old:
-        atomic_write(f'{config_path}.bak', old.read())
-
-    # Set up sections in empty config file
-    for key, val in config.items():
-        section = section_for_key(key)
-        assert section is not None
-        
-        section_name = section.toml_section_header
-        
-        if section_name in config_file:
-            existing_config = dict(config_file[section_name])
-        else:
-            existing_config = {}
-        
-        config_file[section_name] = benedict({**existing_config, key: val})
-        section.update_in_place(warn=False, persist=False, **{key: val})
-
-    with open(config_path, 'w+', encoding='utf-8') as new:
-        config_file.write(new)
-
-    updated_config = {}
-    try:
-        # validate the updated_config by attempting to re-parse it
-        updated_config = {**load_all_config(), **archivebox.pm.hook.get_FLAT_CONFIG()}
-    except BaseException:                                                       # lgtm [py/catch-base-exception]
-        # something went horribly wrong, revert to the previous version
-        with open(f'{config_path}.bak', 'r', encoding='utf-8') as old:
-            atomic_write(config_path, old.read())
-
-        raise
-
-    if os.access(f'{config_path}.bak', os.F_OK):
-        os.remove(f'{config_path}.bak')
-
-    return benedict({
-        key.upper(): updated_config.get(key.upper())
-        for key in config.keys()
-    })
-
-
-
-def load_config(defaults: Dict[str, Any],
-                config: Optional[benedict]=None,
-                out_dir: Optional[str]=None,
-                env_vars: Optional[os._Environ]=None,
-                config_file_vars: Optional[Dict[str, str]]=None) -> benedict:
-
-    env_vars = env_vars or os.environ
-    config_file_vars = config_file_vars or load_config_file()
-
-    extended_config = benedict(config.copy() if config else {})
-    for key, default in defaults.items():
-        try:
-            # print('LOADING CONFIG KEY:', key, 'DEFAULT=', default)
-            extended_config[key] = load_config_val(
-                key,
-                default=default['default'],
-                type=default.get('type'),
-                aliases=default.get('aliases'),
-                config=extended_config,
-                env_vars=env_vars,
-                config_file_vars=config_file_vars,
-            )
-        except KeyboardInterrupt:
-            raise SystemExit(0)
-        except Exception as e:
-            stderr()
-            stderr(f'[X] Error while loading configuration value: {key}', color='red', config=extended_config)
-            stderr('    {}: {}'.format(e.__class__.__name__, e))
-            stderr()
-            stderr('    Check your config for mistakes and try again (your archive data is unaffected).')
-            stderr()
-            stderr('    For config documentation and examples see:')
-            stderr('        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration')
-            stderr()
-            # raise
-            # raise SystemExit(2)
-
-    return benedict(extended_config)
-
-def load_all_config():
-    import abx
-    
-    flat_config = benedict()
-    
-    for config_section in abx.pm.hook.get_CONFIGS().values():
-        config_section.__init__()
-        flat_config.update(config_section.model_dump())
-        
-    return flat_config
-
@@ -1,188 +0,0 @@
-__package__ = 'archivebox.config'
-
-import re
-import sys
-import shutil
-from typing import Dict, Optional, List
-from pathlib import Path
-
-from rich import print
-from pydantic import Field, field_validator
-from django.utils.crypto import get_random_string
-
-from abx_spec_config.base_configset import BaseConfigSet
-
-from .constants import CONSTANTS
-from .version import get_COMMIT_HASH, get_BUILD_TIME, VERSION
-from .permissions import IN_DOCKER
-
-###################### Config ##########################
-
-
-class ShellConfig(BaseConfigSet):
-    DEBUG: bool                         = Field(default=lambda: '--debug' in sys.argv)
-    
-    IS_TTY: bool                        = Field(default=sys.stdout.isatty())
-    USE_COLOR: bool                     = Field(default=lambda c: c.IS_TTY)
-    SHOW_PROGRESS: bool                 = Field(default=lambda c: c.IS_TTY)
-    
-    IN_DOCKER: bool                     = Field(default=IN_DOCKER)
-    IN_QEMU: bool                       = Field(default=False)
-
-    ANSI: Dict[str, str]                = Field(default=lambda c: CONSTANTS.DEFAULT_CLI_COLORS if c.USE_COLOR else CONSTANTS.DISABLED_CLI_COLORS)
-
-    @property
-    def TERM_WIDTH(self) -> int:
-        if not self.IS_TTY:
-            return 200
-        return shutil.get_terminal_size((140, 10)).columns
-    
-    @property
-    def COMMIT_HASH(self) -> Optional[str]:
-        return get_COMMIT_HASH()
-    
-    @property
-    def BUILD_TIME(self) -> str:
-        return get_BUILD_TIME()
- 
-
-SHELL_CONFIG = ShellConfig()
-
-
-class StorageConfig(BaseConfigSet):
-    # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
-    # must be a short path due to unix path length restrictions for socket files (<100 chars)
-    # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
-    TMP_DIR: Path                       = Field(default=CONSTANTS.DEFAULT_TMP_DIR)
-    
-    # LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
-    # must be able to contain executable binaries (up to 5GB size)
-    # should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
-    LIB_DIR: Path                       = Field(default=CONSTANTS.DEFAULT_LIB_DIR)
-    
-    OUTPUT_PERMISSIONS: str             = Field(default='644')
-    RESTRICT_FILE_NAMES: str            = Field(default='windows')
-    ENFORCE_ATOMIC_WRITES: bool         = Field(default=True)
-    
-    # not supposed to be user settable:
-    DIR_OUTPUT_PERMISSIONS: str         = Field(default=lambda c: c['OUTPUT_PERMISSIONS'].replace('6', '7').replace('4', '5'))
-
-
-STORAGE_CONFIG = StorageConfig()
-
-
-class GeneralConfig(BaseConfigSet):
-    TAG_SEPARATOR_PATTERN: str          = Field(default=r'[,]')
-
-GENERAL_CONFIG = GeneralConfig()
-
-
-class ServerConfig(BaseConfigSet):
-    SECRET_KEY: str                     = Field(default=lambda: get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789_'))
-    BIND_ADDR: str                      = Field(default=lambda: ['127.0.0.1:8000', '0.0.0.0:8000'][SHELL_CONFIG.IN_DOCKER])
-    ALLOWED_HOSTS: str                  = Field(default='*')
-    CSRF_TRUSTED_ORIGINS: str           = Field(default=lambda c: 'http://localhost:8000,http://127.0.0.1:8000,http://0.0.0.0:8000,http://{}'.format(c.BIND_ADDR))
-    
-    SNAPSHOTS_PER_PAGE: int             = Field(default=40)
-    PREVIEW_ORIGINALS: bool             = Field(default=True)
-    FOOTER_INFO: str                    = Field(default='Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.')
-    # CUSTOM_TEMPLATES_DIR: Path          = Field(default=None)  # this is now a constant
-
-    PUBLIC_INDEX: bool                  = Field(default=True)
-    PUBLIC_SNAPSHOTS: bool              = Field(default=True)
-    PUBLIC_ADD_VIEW: bool               = Field(default=False)
-    
-    ADMIN_USERNAME: str                 = Field(default=None)
-    ADMIN_PASSWORD: str                 = Field(default=None)
-    
-    REVERSE_PROXY_USER_HEADER: str      = Field(default='Remote-User')
-    REVERSE_PROXY_WHITELIST: str        = Field(default='')
-    LOGOUT_REDIRECT_URL: str            = Field(default='/')
-    
-SERVER_CONFIG = ServerConfig()
-
-
-class ArchivingConfig(BaseConfigSet):
-    ONLY_NEW: bool                        = Field(default=True)
-    
-    TIMEOUT: int                          = Field(default=60)
-    MEDIA_TIMEOUT: int                    = Field(default=3600)
-
-    MEDIA_MAX_SIZE: str                   = Field(default='750m')
-    RESOLUTION: str                       = Field(default='1440,2000')
-    CHECK_SSL_VALIDITY: bool              = Field(default=True)
-    USER_AGENT: str                       = Field(default=f'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)')
-    COOKIES_FILE: Path | None             = Field(default=None)
-    
-    URL_DENYLIST: str                     = Field(default=r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$', alias='URL_BLACKLIST')
-    URL_ALLOWLIST: str | None             = Field(default=None, alias='URL_WHITELIST')
-    
-    SAVE_ALLOWLIST: Dict[str, List[str]]  = Field(default={})  # mapping of regex patterns to list of archive methods
-    SAVE_DENYLIST: Dict[str, List[str]]   = Field(default={})
-    
-    # GIT_DOMAINS: str                    = Field(default='github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht')
-    # WGET_USER_AGENT: str                = Field(default=lambda c: c['USER_AGENT'] + ' wget/{WGET_VERSION}')
-    # CURL_USER_AGENT: str                = Field(default=lambda c: c['USER_AGENT'] + ' curl/{CURL_VERSION}')
-    # CHROME_USER_AGENT: str              = Field(default=lambda c: c['USER_AGENT'])
-    # CHROME_USER_DATA_DIR: str | None    = Field(default=None)
-    # CHROME_TIMEOUT: int                 = Field(default=0)
-    # CHROME_HEADLESS: bool               = Field(default=True)
-    # CHROME_SANDBOX: bool                = Field(default=lambda: not SHELL_CONFIG.IN_DOCKER)
-
-    def validate(self):
-        if int(self.TIMEOUT) < 5:
-            print(f'[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={self.TIMEOUT} seconds)[/red]', file=sys.stderr)
-            print('    You must allow *at least* 5 seconds for indexing and archive methods to run succesfully.', file=sys.stderr)
-            print('    (Setting it to somewhere between 30 and 3000 seconds is recommended)', file=sys.stderr)
-            print(file=sys.stderr)
-            print('    If you want to make ArchiveBox run faster, disable specific archive methods instead:', file=sys.stderr)
-            print('        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles', file=sys.stderr)
-            print(file=sys.stderr)
-    
-    @field_validator('CHECK_SSL_VALIDITY', mode='after')
-    def validate_check_ssl_validity(cls, v):
-        """SIDE EFFECT: disable "you really shouldnt disable ssl" warnings emitted by requests"""
-        if not v:
-            import requests
-            import urllib3
-            requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
-            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-        return v
-    
-    @property
-    def URL_ALLOWLIST_PTN(self) -> re.Pattern | None:
-        return re.compile(self.URL_ALLOWLIST, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS) if self.URL_ALLOWLIST else None
-    
-    @property
-    def URL_DENYLIST_PTN(self) -> re.Pattern:
-        return re.compile(self.URL_DENYLIST, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS)
-    
-    @property
-    def SAVE_ALLOWLIST_PTNS(self) -> Dict[re.Pattern, List[str]]:
-        return {
-            # regexp: methods list
-            re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val
-            for key, val in self.SAVE_ALLOWLIST.items()
-        } if self.SAVE_ALLOWLIST else {}
-    
-    @property
-    def SAVE_DENYLIST_PTNS(self) -> Dict[re.Pattern, List[str]]:
-        return {
-            # regexp: methods list
-            re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val
-            for key, val in self.SAVE_DENYLIST.items()
-        } if self.SAVE_DENYLIST else {}
-
-ARCHIVING_CONFIG = ArchivingConfig()
-
-
-class SearchBackendConfig(BaseConfigSet):
-    USE_INDEXING_BACKEND: bool          = Field(default=True)
-    USE_SEARCHING_BACKEND: bool         = Field(default=True)
-    
-    SEARCH_BACKEND_ENGINE: str          = Field(default='ripgrep')
-    SEARCH_PROCESS_HTML: bool           = Field(default=True)
-    SEARCH_BACKEND_TIMEOUT: int         = Field(default=10)
-
-SEARCH_BACKEND_CONFIG = SearchBackendConfig()
-
@@ -1,233 +0,0 @@
-"""
-Constants are for things that never change at runtime.
-(but they can change from run-to-run or machine-to-machine)
-
-DATA_DIR will never change at runtime, but you can run
-archivebox from inside a different DATA_DIR on the same machine.
-
-This is loaded very early in the archivebox startup flow, so nothing in this file 
-or imported from this file should import anything from archivebox.config.common, 
-django, other INSTALLED_APPS, or anything else that is not in a standard library.
-"""
-
-__package__ = 'archivebox.config'
-
-import re
-import sys
-
-from typing import Dict
-from pathlib import Path
-from collections.abc import Mapping
-
-from benedict import benedict
-
-from ..misc.logging import DEFAULT_CLI_COLORS
-
-from .paths import (
-    PACKAGE_DIR,
-    DATA_DIR,
-    ARCHIVE_DIR,
-    get_collection_id,
-    get_machine_id,
-    get_machine_type,
-)
-from .permissions import (
-    IS_ROOT,
-    IN_DOCKER,
-    RUNNING_AS_UID,
-    RUNNING_AS_GID,
-    DEFAULT_PUID,
-    DEFAULT_PGID,
-    ARCHIVEBOX_USER,
-    ARCHIVEBOX_GROUP,
-)
-from .version import detect_installed_version
-
-###################### Config ##########################
-
-
-class ConstantsDict(Mapping):
-    PACKAGE_DIR: Path                   = PACKAGE_DIR
-    DATA_DIR: Path                      = DATA_DIR
-    ARCHIVE_DIR: Path                   = ARCHIVE_DIR
-    
-    MACHINE_TYPE: str                   = get_machine_type()
-    MACHINE_ID: str                     = get_machine_id()
-    COLLECTION_ID: str                  = get_collection_id(DATA_DIR)
-    
-    # Host system
-    VERSION: str                        = detect_installed_version(PACKAGE_DIR)
-    IN_DOCKER: bool                     = IN_DOCKER
-    
-    # Permissions
-    IS_ROOT: bool                       = IS_ROOT
-    ARCHIVEBOX_USER: int                = ARCHIVEBOX_USER
-    ARCHIVEBOX_GROUP: int               = ARCHIVEBOX_GROUP
-    RUNNING_AS_UID: int                 = RUNNING_AS_UID
-    RUNNING_AS_GID: int                 = RUNNING_AS_GID
-    DEFAULT_PUID: int                   = DEFAULT_PUID
-    DEFAULT_PGID: int                   = DEFAULT_PGID
-    IS_INSIDE_VENV: bool                = sys.prefix != sys.base_prefix
-    
-    # Source code dirs
-    PACKAGE_DIR_NAME: str               = PACKAGE_DIR.name
-    TEMPLATES_DIR_NAME: str             = 'templates'
-    TEMPLATES_DIR: Path                 = PACKAGE_DIR / TEMPLATES_DIR_NAME
-    STATIC_DIR_NAME: str                = 'static'
-    STATIC_DIR: Path                    = TEMPLATES_DIR / STATIC_DIR_NAME
-
-    # Data dirs
-    ARCHIVE_DIR_NAME: str               = 'archive'
-    SOURCES_DIR_NAME: str               = 'sources'
-    PERSONAS_DIR_NAME: str              = 'personas'
-    CRONTABS_DIR_NAME: str              = 'crontabs'
-    CACHE_DIR_NAME: str                 = 'cache'
-    LOGS_DIR_NAME: str                  = 'logs'
-    USER_PLUGINS_DIR_NAME: str          = 'user_plugins'
-    CUSTOM_TEMPLATES_DIR_NAME: str      = 'user_templates'
-    ARCHIVE_DIR: Path                   = DATA_DIR / ARCHIVE_DIR_NAME
-    SOURCES_DIR: Path                   = DATA_DIR / SOURCES_DIR_NAME
-    PERSONAS_DIR: Path                  = DATA_DIR / PERSONAS_DIR_NAME
-    LOGS_DIR: Path                      = DATA_DIR / LOGS_DIR_NAME
-    CACHE_DIR: Path                     = DATA_DIR / CACHE_DIR_NAME
-    CUSTOM_TEMPLATES_DIR: Path          = DATA_DIR / CUSTOM_TEMPLATES_DIR_NAME
-    USER_PLUGINS_DIR: Path              = DATA_DIR / USER_PLUGINS_DIR_NAME
-
-    # Data dir files
-    CONFIG_FILENAME: str                = 'ArchiveBox.conf'
-    SQL_INDEX_FILENAME: str             = 'index.sqlite3'
-    QUEUE_DATABASE_FILENAME: str        = 'queue.sqlite3'
-    CONFIG_FILE: Path                   = DATA_DIR / CONFIG_FILENAME
-    DATABASE_FILE: Path                 = DATA_DIR / SQL_INDEX_FILENAME
-    QUEUE_DATABASE_FILE: Path           = DATA_DIR / QUEUE_DATABASE_FILENAME
-    
-    JSON_INDEX_FILENAME: str            = 'index.json'
-    HTML_INDEX_FILENAME: str            = 'index.html'
-    ROBOTS_TXT_FILENAME: str            = 'robots.txt'
-    FAVICON_FILENAME: str               = 'favicon.ico'
-    
-    # Runtime dirs
-    TMP_DIR_NAME: str                   = 'tmp'
-    DEFAULT_TMP_DIR: Path               = DATA_DIR / TMP_DIR_NAME / MACHINE_ID    # ./data/tmp/abc3244323
-    
-    LIB_DIR_NAME: str                   = 'lib'
-    DEFAULT_LIB_DIR: Path               = DATA_DIR / LIB_DIR_NAME / MACHINE_TYPE  # ./data/lib/arm64-linux-docker
-
-    # Config constants
-    TIMEZONE: str                       = 'UTC'
-    DEFAULT_CLI_COLORS: Dict[str, str]  = DEFAULT_CLI_COLORS
-    DISABLED_CLI_COLORS: Dict[str, str] = benedict({k: '' for k in DEFAULT_CLI_COLORS})
-
-    ALLOWDENYLIST_REGEX_FLAGS: int      = re.IGNORECASE | re.UNICODE | re.MULTILINE
-
-    STATICFILE_EXTENSIONS: frozenset[str] = frozenset((
-        # 99.999% of the time, URLs ending in these extensions are static files
-        # that can be downloaded as-is, not html pages that need to be rendered
-        'gif', 'jpeg', 'jpg', 'png', 'tif', 'tiff', 'wbmp', 'ico', 'jng', 'bmp',
-        'svg', 'svgz', 'webp', 'ps', 'eps', 'ai',
-        'mp3', 'mp4', 'm4a', 'mpeg', 'mpg', 'mkv', 'mov', 'webm', 'm4v',
-        'flv', 'wmv', 'avi', 'ogg', 'ts', 'm3u8',
-        'pdf', 'txt', 'rtf', 'rtfd', 'doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx',
-        'atom', 'rss', 'css', 'js', 'json',
-        'dmg', 'iso', 'img',
-        'rar', 'war', 'hqx', 'zip', 'gz', 'bz2', '7z',
-
-        # Less common extensions to consider adding later
-        # jar, swf, bin, com, exe, dll, deb
-        # ear, hqx, eot, wmlc, kml, kmz, cco, jardiff, jnlp, run, msi, msp, msm,
-        # pl pm, prc pdb, rar, rpm, sea, sit, tcl tk, der, pem, crt, xpi, xspf,
-        # ra, mng, asx, asf, 3gpp, 3gp, mid, midi, kar, jad, wml, htc, mml
-
-        # These are always treated as pages, not as static files, never add them:
-        # html, htm, shtml, xhtml, xml, aspx, php, cgi
-    ))
-
-    PIP_RELATED_NAMES: frozenset[str] = frozenset((
-        ".venv",
-        "venv",
-        "virtualenv",
-        ".virtualenv",
-    ))
-    NPM_RELATED_NAMES: frozenset[str] = frozenset((
-        "node_modules",
-        "package.json",
-        "package-lock.json",
-        "yarn.lock",
-    ))
-
-    # When initializing archivebox in a new directory, we check to make sure the dir is
-    # actually empty so that we dont clobber someone's home directory or desktop by accident.
-    # These files are exceptions to the is_empty check when we're trying to init a new dir,
-    # as they could be from a previous archivebox version, system artifacts, dependencies, etc.
-    ALLOWED_IN_DATA_DIR: frozenset[str] = frozenset((
-        *PIP_RELATED_NAMES,
-        *NPM_RELATED_NAMES,
-        
-        ### Dirs:
-        ARCHIVE_DIR_NAME,
-        SOURCES_DIR_NAME,
-        LOGS_DIR_NAME,
-        CACHE_DIR_NAME,
-        LIB_DIR_NAME,
-        TMP_DIR_NAME,
-        PERSONAS_DIR_NAME,
-        CUSTOM_TEMPLATES_DIR_NAME,
-        USER_PLUGINS_DIR_NAME,
-        CRONTABS_DIR_NAME,
-        "static",                # created by old static exports <v0.6.0
-        "sonic",                 # created by docker bind mount / sonic FTS process
-        ".git",
-        ".svn",
-        
-        ### Files:
-        CONFIG_FILENAME,
-        SQL_INDEX_FILENAME,
-        f"{SQL_INDEX_FILENAME}-wal",
-        f"{SQL_INDEX_FILENAME}-shm",
-        QUEUE_DATABASE_FILENAME,
-        f"{QUEUE_DATABASE_FILENAME}-wal",
-        f"{QUEUE_DATABASE_FILENAME}-shm",
-        "search.sqlite3",
-        JSON_INDEX_FILENAME,
-        HTML_INDEX_FILENAME,
-        ROBOTS_TXT_FILENAME,
-        FAVICON_FILENAME,
-        CONFIG_FILENAME,
-        f"{CONFIG_FILENAME}.bak",
-        f".{CONFIG_FILENAME}.bak",
-        "static_index.json",
-        ".DS_Store",
-        ".gitignore",
-        "lost+found",
-        ".DS_Store",
-        ".env",
-        ".collection_id",
-        ".archivebox_id",
-        "Dockerfile",
-    ))
-        
-
-    @classmethod
-    def __getitem__(cls, key: str):
-        # so it behaves like a dict[key] == dict.key or object attr
-        return getattr(cls, key)
-    
-    @classmethod
-    def __benedict__(cls):
-        # when casting to benedict, only include uppercase keys that don't start with an underscore
-        return benedict({key: value for key, value in cls.__dict__.items() if key.isupper() and not key.startswith('_')})
-    
-    @classmethod
-    def __len__(cls):
-        return len(cls.__benedict__())
-
-    @classmethod
-    def __iter__(cls):
-        return iter(cls.__benedict__())
-
-CONSTANTS = ConstantsDict()
-CONSTANTS_CONFIG = CONSTANTS.__benedict__()
-
-# add all key: values to globals() for easier importing, e.g.:
-# from archivebox.config.constants import IS_ROOT, PERSONAS_DIR, ...
-# globals().update(CONSTANTS)
@@ -1,176 +0,0 @@
-__package__ = 'archivebox.config'
-
-import os
-import sys
-
-from datetime import datetime, timezone
-
-from rich.progress import Progress
-from rich.console import Console
-
-import django
-
-from . import CONSTANTS
-
-from .common import SHELL_CONFIG
-from ..misc import logging
-
-
-if not SHELL_CONFIG.USE_COLOR:
-    os.environ['NO_COLOR'] = '1'
-if not SHELL_CONFIG.SHOW_PROGRESS:
-    os.environ['TERM'] = 'dumb'
-
-# recreate rich console obj based on new config values
-STDOUT = CONSOLE = Console()
-STDERR = Console(stderr=True)
-logging.CONSOLE = CONSOLE
-
-
-INITIAL_STARTUP_PROGRESS = None
-INITIAL_STARTUP_PROGRESS_TASK = 0
-
-def bump_startup_progress_bar(advance=1):
-    global INITIAL_STARTUP_PROGRESS
-    global INITIAL_STARTUP_PROGRESS_TASK
-    if INITIAL_STARTUP_PROGRESS:
-        INITIAL_STARTUP_PROGRESS.update(INITIAL_STARTUP_PROGRESS_TASK, advance=advance)   # type: ignore
-
-
-def setup_django_minimal():
-    # sys.path.append(str(CONSTANTS.PACKAGE_DIR))
-    # os.environ.setdefault('ARCHIVEBOX_DATA_DIR', str(CONSTANTS.DATA_DIR))
-    # os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
-    # django.setup()
-    raise Exception('dont use this anymore')
-
-DJANGO_SET_UP = False
-
-
-def setup_django(check_db=False, in_memory_db=False) -> None:
-    from rich.panel import Panel
-    
-    global INITIAL_STARTUP_PROGRESS
-    global INITIAL_STARTUP_PROGRESS_TASK
-    global DJANGO_SET_UP
-
-    if DJANGO_SET_UP:
-        # raise Exception('django is already set up!')
-        # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
-        return
-
-    with Progress(transient=True, expand=True, console=STDERR) as INITIAL_STARTUP_PROGRESS:
-        INITIAL_STARTUP_PROGRESS_TASK = INITIAL_STARTUP_PROGRESS.add_task("[green]Loading modules...", total=25, visible=False)
-        
-        from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, SudoPermission
-    
-        # if running as root, chown the data dir to the archivebox user to make sure it's accessible to the archivebox user
-        if IS_ROOT and ARCHIVEBOX_USER != 0:
-            with SudoPermission(uid=0):
-                # running as root is a special case where it's ok to be a bit slower
-                # make sure data dir is always owned by the correct user
-                os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{CONSTANTS.DATA_DIR}" 2>/dev/null')
-                os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{CONSTANTS.DATA_DIR}"/* 2>/dev/null')
-
-        bump_startup_progress_bar()
-        try:
-            from django.core.management import call_command
-                
-            bump_startup_progress_bar()
-
-            if in_memory_db:
-                raise Exception('dont use this anymore')
-            
-                # some commands (e.g. oneshot) dont store a long-lived sqlite3 db file on disk.
-                # in those cases we create a temporary in-memory db and run the migrations
-                # immediately to get a usable in-memory-database at startup
-                os.environ.setdefault("ARCHIVEBOX_DATABASE_NAME", ":memory:")
-                django.setup()
-                
-                bump_startup_progress_bar()
-                call_command("migrate", interactive=False, verbosity=0)
-            else:
-                # Otherwise use default sqlite3 file-based database and initialize django
-                # without running migrations automatically (user runs them manually by calling init)
-                try:
-                    django.setup()
-                except Exception as e:
-                    bump_startup_progress_bar(advance=1000)
-                    
-                    is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ('help', 'version', '--help', '--version'))
-                    if not is_using_meta_cmd:
-                        # show error message to user only if they're not running a meta command / just trying to get help
-                        STDERR.print()
-                        STDERR.print(Panel(
-                            f'\n[red]{e.__class__.__name__}[/red]: [yellow]{e}[/yellow]\nPlease check your config and [blue]DATA_DIR[/blue] permissions.\n',
-                            title='\n\n[red][X] Error while trying to load database![/red]',
-                            subtitle='[grey53]NO WRITES CAN BE PERFORMED[/grey53]',
-                            expand=False,
-                            style='bold red',
-                        ))
-                        STDERR.print()
-                        STDERR.print_exception(show_locals=False)
-                    return
-            
-            bump_startup_progress_bar()
-
-            from django.conf import settings
-            
-            # log startup message to the error log
-            with open(settings.ERROR_LOG, "a", encoding='utf-8') as f:
-                command = ' '.join(sys.argv)
-                ts = datetime.now(timezone.utc).strftime('%Y-%m-%d__%H:%M:%S')
-                f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={SHELL_CONFIG.IN_DOCKER} IS_TTY={SHELL_CONFIG.IS_TTY}\n")
-
-            if check_db:
-                # make sure the data dir is owned by a non-root user
-                if CONSTANTS.DATA_DIR.stat().st_uid == 0:
-                    STDERR.print('[red][X] Error: ArchiveBox DATA_DIR cannot be owned by root![/red]')
-                    STDERR.print(f'    {CONSTANTS.DATA_DIR}')
-                    STDERR.print()
-                    STDERR.print('[violet]Hint:[/violet] Are you running archivebox in the right folder? (and as a non-root user?)')
-                    STDERR.print('    cd path/to/your/archive/data')
-                    STDERR.print('    archivebox [command]')
-                    STDERR.print()
-                    raise SystemExit(9)
-                
-                # Create cache table in DB if needed
-                try:
-                    from django.core.cache import cache
-                    cache.get('test', None)
-                except django.db.utils.OperationalError:
-                    call_command("createcachetable", verbosity=0)
-
-                bump_startup_progress_bar()
-
-                # if archivebox gets imported multiple times, we have to close
-                # the sqlite3 whenever we init from scratch to avoid multiple threads
-                # sharing the same connection by accident
-                from django.db import connections
-                for conn in connections.all():
-                    conn.close_if_unusable_or_obsolete()
-
-                sql_index_path = CONSTANTS.DATABASE_FILE
-                assert os.access(sql_index_path, os.F_OK), (
-                    f'No database file {sql_index_path} found in: {CONSTANTS.DATA_DIR} (Are you in an ArchiveBox collection directory?)')
-
-                bump_startup_progress_bar()
-
-                # https://docs.pydantic.dev/logfire/integrations/django/ Logfire Debugging
-                # if settings.DEBUG_LOGFIRE:
-                #     from opentelemetry.instrumentation.sqlite3 import SQLite3Instrumentor
-                #     SQLite3Instrumentor().instrument()
-
-                #     import logfire
-
-                #     logfire.configure()
-                #     logfire.instrument_django(is_sql_commentor_enabled=True)
-                #     logfire.info(f'Started ArchiveBox v{CONSTANTS.VERSION}', argv=sys.argv)
-
-        except KeyboardInterrupt:
-            raise SystemExit(2)
-        
-    DJANGO_SET_UP = True
-
-    INITIAL_STARTUP_PROGRESS = None
-    INITIAL_STARTUP_PROGRESS_TASK = None
@@ -1,413 +0,0 @@
-__package__ = 'archivebox.config'
-
-import os
-import socket
-import hashlib
-import tempfile
-import platform
-from pathlib import Path
-from functools import cache
-from datetime import datetime
-
-from benedict import benedict
-
-from .permissions import SudoPermission, IS_ROOT, ARCHIVEBOX_USER
-
-#############################################################################################
-
-PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent    # archivebox source code dir
-DATA_DIR: Path = Path(os.getcwd()).resolve()                  # archivebox user data dir
-ARCHIVE_DIR: Path = DATA_DIR / 'archive'                      # archivebox snapshot data dir
-
-IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE', 'yes')
-
-DATABASE_FILE = DATA_DIR / 'index.sqlite3'
-
-#############################################################################################
-
-def _get_collection_id(DATA_DIR=DATA_DIR, force_create=False) -> str:
-    collection_id_file = DATA_DIR / '.archivebox_id'
-    
-    try:
-        return collection_id_file.read_text().strip()
-    except (OSError, FileNotFoundError, PermissionError):
-        pass
-    
-    # hash the machine_id + collection dir path + creation time to get a unique collection_id
-    machine_id = get_machine_id()
-    collection_path = DATA_DIR.resolve()
-    try:
-        creation_date = DATA_DIR.stat().st_ctime
-    except Exception:
-        creation_date = datetime.now().isoformat()
-    collection_id = hashlib.sha256(f'{machine_id}:{collection_path}@{creation_date}'.encode()).hexdigest()[:8]
-    
-    try:
-        # only persist collection_id file if we already have an index.sqlite3 file present
-        # otherwise we might be running in a directory that is not a collection, no point creating cruft files
-        collection_is_active = os.path.isfile(DATABASE_FILE) and os.path.isdir(ARCHIVE_DIR) and os.access(DATA_DIR, os.W_OK)
-        if collection_is_active or force_create:
-            collection_id_file.write_text(collection_id)
-            
-            # if we're running as root right now, make sure the collection_id file is owned by the archivebox user
-            if IS_ROOT:
-                with SudoPermission(uid=0):
-                    if ARCHIVEBOX_USER == 0:
-                        os.system(f'chmod 777 "{collection_id_file}"')
-                    else:    
-                        os.system(f'chown {ARCHIVEBOX_USER} "{collection_id_file}"')
-    except (OSError, FileNotFoundError, PermissionError):
-        pass
-    return collection_id
-
-@cache
-def get_collection_id(DATA_DIR=DATA_DIR) -> str:
-    """Get a short, stable, unique ID for the current collection (e.g. abc45678)"""
-    return _get_collection_id(DATA_DIR=DATA_DIR)
-
-@cache
-def get_machine_id() -> str:
-    """Get a short, stable, unique ID for the current machine (e.g. abc45678)"""
-    
-    MACHINE_ID = 'unknown'
-    try:
-        import machineid
-        MACHINE_ID = machineid.hashed_id('archivebox')[:8]
-    except Exception:
-        try:
-            import uuid
-            import hashlib
-            MACHINE_ID = hashlib.sha256(str(uuid.getnode()).encode()).hexdigest()[:8]
-        except Exception:
-            pass
-    return MACHINE_ID
-
-@cache
-def get_machine_type() -> str:
-    """Get a short, stable, unique type identifier for the current machine (e.g. linux-x86_64-docker)"""
-    
-    OS: str                             = platform.system().lower()    # darwin, linux, etc.
-    ARCH: str                           = platform.machine().lower()   # arm64, x86_64, aarch64, etc.
-    LIB_DIR_SCOPE: str                  = f'{ARCH}-{OS}-docker' if IN_DOCKER else f'{ARCH}-{OS}'
-    return LIB_DIR_SCOPE
-
-
-def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = None, fallback=True, chown=True) -> bool:
-    """Check if a given directory is writable by a specific user and group (fallback=try as current user is unable to check with provided uid)"""
-    current_uid, current_gid = os.geteuid(), os.getegid()
-    uid, gid = uid or current_uid, gid or current_gid
-
-    test_file = dir_path / '.permissions_test'
-    try:
-        with SudoPermission(uid=uid, fallback=fallback):
-            test_file.exists()
-            test_file.write_text(f'Checking if PUID={uid} PGID={gid} can write to dir')
-            test_file.unlink()
-            return True
-    except (IOError, OSError, PermissionError):
-        if chown:    
-            # try fixing it using sudo permissions
-            with SudoPermission(uid=uid, fallback=fallback):
-                os.system(f'chown {uid}:{gid} "{dir_path}" 2>/dev/null')
-            return dir_is_writable(dir_path, uid=uid, gid=gid, fallback=fallback, chown=False)
-    return False
-
-def assert_dir_can_contain_unix_sockets(dir_path: Path) -> bool:
-    """Check if a given directory can contain unix sockets (e.g. /tmp/supervisord.sock)"""
-    from archivebox.logging_util import pretty_path
-    
-    try:
-        socket_path = str(dir_path / '.test_socket.sock')
-        s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-        try:
-            os.remove(socket_path)
-        except OSError:
-            pass
-        s.bind(socket_path)
-        s.close()
-        try:
-            os.remove(socket_path)
-        except OSError:
-            pass
-    except Exception as e:
-        raise Exception(f'ArchiveBox failed to create a test UNIX socket file in {pretty_path(dir_path, color=False)}') from e
-    
-    return True
-
-
-def create_and_chown_dir(dir_path: Path) -> None:
-    with SudoPermission(uid=0, fallback=True):
-        dir_path.mkdir(parents=True, exist_ok=True)
-        os.system(f'chown {ARCHIVEBOX_USER} "{dir_path}" 2>/dev/null')
-        os.system(f'chown {ARCHIVEBOX_USER} "{dir_path}"/* 2>/dev/null &')
-
-@cache
-def get_or_create_working_tmp_dir(autofix=True, quiet=False):
-    from archivebox import CONSTANTS
-    from archivebox.config.common import STORAGE_CONFIG
-    from archivebox.misc.checks import check_tmp_dir
-
-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        STORAGE_CONFIG.TMP_DIR,                                                # <user-specified>
-        CONSTANTS.DEFAULT_TMP_DIR,                                             # ./data/tmp/<machine_id>
-        Path('/var/run/archivebox') / get_collection_id(),                     # /var/run/archivebox/abc5d8512
-        Path('/tmp') / 'archivebox' / get_collection_id(),                     # /tmp/archivebox/abc5d8512
-        Path('~/.tmp/archivebox').expanduser() / get_collection_id(),          # ~/.tmp/archivebox/abc5d8512
-        Path(tempfile.gettempdir()) / 'archivebox' / get_collection_id(),      # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d8512
-        Path(tempfile.gettempdir()) / 'archivebox' / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d
-        Path(tempfile.gettempdir()) / 'abx' / get_collection_id()[:4],         # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/abx/abc5
-    ]
-    for candidate in CANDIDATES:
-        try:
-            create_and_chown_dir(candidate)
-        except Exception:
-            pass
-        if check_tmp_dir(candidate, throw=False, quiet=True, must_exist=True):
-            if autofix and STORAGE_CONFIG.TMP_DIR != candidate:
-                STORAGE_CONFIG.update_in_place(TMP_DIR=candidate, warn=not quiet)
-            return candidate
-    
-    if not quiet:
-        raise OSError(f'ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!')
-
-@cache
-def get_or_create_working_lib_dir(autofix=True, quiet=False):
-    from archivebox import CONSTANTS
-    from archivebox.config.common import STORAGE_CONFIG
-    from archivebox.misc.checks import check_lib_dir
-    
-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        STORAGE_CONFIG.LIB_DIR,                                                   # <user-specified>
-        CONSTANTS.DEFAULT_LIB_DIR,                                                # ./data/lib/arm64-linux-docker
-        Path('/usr/local/share/archivebox') / get_collection_id(),                # /usr/local/share/archivebox/abc5
-        *([Path('/opt/homebrew/share/archivebox') / get_collection_id()] if os.path.isfile('/opt/homebrew/bin/archivebox') else []),  # /opt/homebrew/share/archivebox/abc5
-        Path('~/.local/share/archivebox').expanduser() / get_collection_id(),     # ~/.local/share/archivebox/abc5
-    ]
-    
-    for candidate in CANDIDATES:
-        try:
-            create_and_chown_dir(candidate)
-        except Exception:
-            pass
-        if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True):
-            if autofix and STORAGE_CONFIG.LIB_DIR != candidate:
-                STORAGE_CONFIG.update_in_place(LIB_DIR=candidate, warn=not quiet)
-            return candidate
-    
-    if not quiet:
-        raise OSError(f'ArchiveBox is unable to find a writable LIB_DIR, tried {CANDIDATES}!')
-
-
-
-@cache
-def get_data_locations():
-    from archivebox.config import CONSTANTS
-    from archivebox.config.common import STORAGE_CONFIG
-    
-    return benedict({
-        "DATA_DIR": {
-            "path": DATA_DIR.resolve(),
-            "enabled": True,
-            "is_valid": os.path.isdir(DATA_DIR) and os.access(DATA_DIR, os.R_OK) and os.access(DATA_DIR, os.W_OK),
-            "is_mount": os.path.ismount(DATA_DIR.resolve()),
-        },
-        "CONFIG_FILE": {
-            "path": CONSTANTS.CONFIG_FILE.resolve(),
-            "enabled": True,
-            "is_valid": os.path.isfile(CONSTANTS.CONFIG_FILE) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.W_OK),
-        },
-        "SQL_INDEX": {
-            "path": DATABASE_FILE.resolve(),
-            "enabled": True,
-            "is_valid": os.path.isfile(DATABASE_FILE) and os.access(DATABASE_FILE, os.R_OK) and os.access(DATABASE_FILE, os.W_OK),
-            "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
-        },
-        "QUEUE_DATABASE": {
-            "path": CONSTANTS.QUEUE_DATABASE_FILE,
-            "enabled": True,
-            "is_valid": os.path.isfile(CONSTANTS.QUEUE_DATABASE_FILE) and os.access(CONSTANTS.QUEUE_DATABASE_FILE, os.R_OK) and os.access(CONSTANTS.QUEUE_DATABASE_FILE, os.W_OK),
-            "is_mount": os.path.ismount(CONSTANTS.QUEUE_DATABASE_FILE),
-        },
-        "ARCHIVE_DIR": {
-            "path": ARCHIVE_DIR.resolve(),
-            "enabled": True,
-            "is_valid": os.path.isdir(ARCHIVE_DIR) and os.access(ARCHIVE_DIR, os.R_OK) and os.access(ARCHIVE_DIR, os.W_OK),
-            "is_mount": os.path.ismount(ARCHIVE_DIR.resolve()),
-        },
-        "SOURCES_DIR": {
-            "path": CONSTANTS.SOURCES_DIR.resolve(),
-            "enabled": True,
-            "is_valid": os.path.isdir(CONSTANTS.SOURCES_DIR) and os.access(CONSTANTS.SOURCES_DIR, os.R_OK) and os.access(CONSTANTS.SOURCES_DIR, os.W_OK),
-        },
-        "PERSONAS_DIR": {
-            "path": CONSTANTS.PERSONAS_DIR.resolve(),
-            "enabled": os.path.isdir(CONSTANTS.PERSONAS_DIR),
-            "is_valid": os.path.isdir(CONSTANTS.PERSONAS_DIR) and os.access(CONSTANTS.PERSONAS_DIR, os.R_OK) and os.access(CONSTANTS.PERSONAS_DIR, os.W_OK),                 # read + write
-        },
-        "LOGS_DIR": {
-            "path": CONSTANTS.LOGS_DIR.resolve(),
-            "enabled": True,
-            "is_valid": os.path.isdir(CONSTANTS.LOGS_DIR) and os.access(CONSTANTS.LOGS_DIR, os.R_OK) and os.access(CONSTANTS.LOGS_DIR, os.W_OK),                             # read + write
-        },
-        'TMP_DIR': {
-            'path': STORAGE_CONFIG.TMP_DIR.resolve(),
-            'enabled': True,
-            'is_valid': os.path.isdir(STORAGE_CONFIG.TMP_DIR) and os.access(STORAGE_CONFIG.TMP_DIR, os.R_OK) and os.access(STORAGE_CONFIG.TMP_DIR, os.W_OK),        # read + write
-        },
-        # "CACHE_DIR": {
-        #     "path": CACHE_DIR.resolve(),
-        #     "enabled": True,
-        #     "is_valid": os.access(CACHE_DIR, os.R_OK) and os.access(CACHE_DIR, os.W_OK),                        # read + write
-        # },
-    })
-
-@cache
-def get_code_locations():
-    from archivebox.config import CONSTANTS
-    from archivebox.config.common import STORAGE_CONFIG
-    
-    return benedict({
-        'PACKAGE_DIR': {
-            'path': (PACKAGE_DIR).resolve(),
-            'enabled': True,
-            'is_valid': os.access(PACKAGE_DIR / '__main__.py', os.X_OK),                                                                  # executable
-        },
-        'TEMPLATES_DIR': {
-            'path': CONSTANTS.TEMPLATES_DIR.resolve(),
-            'enabled': True,
-            'is_valid': os.access(CONSTANTS.STATIC_DIR, os.R_OK) and os.access(CONSTANTS.STATIC_DIR, os.X_OK),                                                # read + list
-        },
-        'CUSTOM_TEMPLATES_DIR': {
-            'path': CONSTANTS.CUSTOM_TEMPLATES_DIR.resolve(),
-            'enabled': os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR),
-            'is_valid': os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK),                                      # read
-        },
-        'USER_PLUGINS_DIR': {
-            'path': CONSTANTS.USER_PLUGINS_DIR.resolve(),
-            'enabled': os.path.isdir(CONSTANTS.USER_PLUGINS_DIR),
-            'is_valid': os.path.isdir(CONSTANTS.USER_PLUGINS_DIR) and os.access(CONSTANTS.USER_PLUGINS_DIR, os.R_OK),                                              # read
-        },
-        'LIB_DIR': {
-            'path': STORAGE_CONFIG.LIB_DIR.resolve(),
-            'enabled': True,
-            'is_valid': os.path.isdir(STORAGE_CONFIG.LIB_DIR) and os.access(STORAGE_CONFIG.LIB_DIR, os.R_OK) and os.access(STORAGE_CONFIG.LIB_DIR, os.W_OK),                      # read + write
-        },
-    })
-
-
-
-# @cache
-# def get_LIB_DIR():
-#     """
-#     - should be shared with other collections on the same host
-#     - must be scoped by CPU architecture, OS family, and archivebox version
-#     - should not be shared with other hosts/archivebox versions
-#     - must be writable by any archivebox user
-#     - should be persistent across reboots
-#     - can be on a docker bin mount but probably shouldnt be
-#     - ok to have a long path (doesnt contain SOCKETS)
-#     """
-#     from .version import detect_installed_version
-    
-#     HOST_DIRS = PlatformDirs(appname='archivebox', appauthor='ArchiveBox', version=detect_installed_version(), opinion=True, ensure_exists=False)
-    
-#     lib_dir = tempfile.gettempdir()
-#     try:
-#         if 'SYSTEM_LIB_DIR' in os.environ:
-#             lib_dir = Path(os.environ['SYSTEM_LIB_DIR'])
-#         else:
-#             with SudoPermission(uid=ARCHIVEBOX_USER, fallback=True):
-#                 lib_dir = HOST_DIRS.site_data_path
-        
-#         # Docker: /usr/local/share/archivebox/0.8.5
-#         # Ubuntu: /usr/local/share/archivebox/0.8.5
-#         # macOS: /Library/Application Support/archivebox
-#         try:
-#             with SudoPermission(uid=0, fallback=True):
-#                 lib_dir.mkdir(parents=True, exist_ok=True)
-#         except PermissionError:
-#             # our user cannot 
-#             lib_dir = HOST_DIRS.user_data_path
-#             lib_dir.mkdir(parents=True, exist_ok=True)
-        
-#         if IS_ROOT or not dir_is_writable(lib_dir, uid=ARCHIVEBOX_USER):
-#             if IS_ROOT:
-#                 # make sure lib dir is owned by the archivebox user, not root
-#                 with SudoPermission(uid=0):
-#                     if ARCHIVEBOX_USER == 0:
-#                         # print(f'[yellow]:warning:  Waring: Creating SYSTEM_LIB_DIR {lib_dir} with mode 777 so that non-root archivebox users can share it.[/yellow] (caches shared libs used by archivebox for performance)', file=sys.stderr)
-#                         os.system(f'chmod -R 777 "{lib_dir}"')
-#                     else:
-#                         os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{lib_dir}"')
-#             else:
-#                 raise PermissionError()
-#     except (PermissionError, AssertionError):
-#         # raise PermissionError(f'SYSTEM_LIB_DIR {lib_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}')
-#         print(f'[red]:cross_mark:  ERROR: SYSTEM_LIB_DIR {lib_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/red]', file=sys.stderr)
-        
-#     return lib_dir
-    
-# @cache
-# def get_TMP_DIR():
-#     """
-#     - must NOT be inside DATA_DIR / inside a docker volume bind mount
-#     - must NOT have a long PATH (UNIX socket path length restrictions)
-#     - must NOT be shared with other collections/hosts
-#     - must be writable by archivebox user & root
-#     - must be cleared on every boot / not persisted
-#     - must be cleared on every archivebox version upgrade
-#     """
-#     from .version import detect_installed_version
-    
-#     HOST_DIRS = PlatformDirs(appname='archivebox', appauthor='ArchiveBox', version=detect_installed_version(), opinion=True, ensure_exists=False)
-    
-#     # print('DATA_DIR OWNED BY:', ARCHIVEBOX_USER, ARCHIVEBOX_GROUP)
-#     # print('RUNNING AS:', self.PUID, self.PGID)
-#     run_dir = tempfile.gettempdir()
-#     try:
-#         if 'SYSTEM_TMP_DIR' in os.environ:
-#             run_dir = Path(os.environ['SYSTEM_TMP_DIR']).resolve() / get_collection_id(DATA_DIR=DATA_DIR)
-#             with SudoPermission(uid=0, fallback=True):
-#                 run_dir.mkdir(parents=True, exist_ok=True)
-#             if not dir_is_writable(run_dir, uid=ARCHIVEBOX_USER):
-#                 if IS_ROOT:
-#                     with SudoPermission(uid=0, fallback=False):
-#                         if ARCHIVEBOX_USER == 0:
-#                             # print(f'[yellow]:warning:  Waring: Creating SYSTEM_TMP_DIR {run_dir} with mode 777 so that non-root archivebox users can access it.[/yellow]', file=sys.stderr)
-#                             os.system(f'chmod -R 777 "{run_dir}"')
-#                         else:
-#                             os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{run_dir}"')
-#                 else:
-#                     raise PermissionError()
-#             assert len(str(run_dir / 'supervisord.conf')) < 95, 'SYSTEM_TMP_DIR path is too long, please set SYSTEM_TMP_DIR env variable to a shorter path (unfortunately unix requires socket paths be < 108 chars)'
-#             return run_dir
-        
-#         run_dir = (HOST_DIRS.site_runtime_path / get_collection_id(DATA_DIR=DATA_DIR)).resolve()
-#         try:
-#             assert len(str(run_dir)) + len('/supervisord.sock') < 95
-#         except AssertionError:
-#             run_dir = Path(tempfile.gettempdir()).resolve() / 'archivebox' / get_collection_id(DATA_DIR=DATA_DIR)
-#             assert len(str(run_dir)) + len('/supervisord.sock') < 95, 'SYSTEM_TMP_DIR path is too long, please set SYSTEM_TMP_DIR env variable to a shorter path (unfortunately unix requires socket paths be < 108 chars)'
-        
-#         with SudoPermission(uid=0, fallback=True):
-#             run_dir.mkdir(parents=True, exist_ok=True)
-            
-#         if IS_ROOT or not dir_is_writable(run_dir, uid=ARCHIVEBOX_USER):
-#             if IS_ROOT:
-#                 with SudoPermission(uid=0):
-#                     if ARCHIVEBOX_USER == 0:
-#                         # print(f'[yellow]:warning:  Waring: Creating SYSTEM_TMP_DIR {run_dir} with mode 777 so that non-root archivebox users can access it.[/yellow]', file=sys.stderr)
-#                         os.system(f'chmod -R 777 "{run_dir}"')
-#                     else:
-#                         os.system(f'chown {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP} "{run_dir}"')
-#             else:
-#                 raise PermissionError()
-            
-#     except (PermissionError, AssertionError):
-#         # raise PermissionError(f'SYSTEM_TMP_DIR {run_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}')
-#         print(f'[red]:cross_mark:  ERROR: SYSTEM_TMP_DIR {run_dir} is not writable by archivebox user {ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/red]', file=sys.stderr)
-        
-#     return run_dir
-
@@ -1,133 +0,0 @@
-__package__ = 'archivebox.config'
-
-import os
-import pwd
-import sys
-
-from rich import print
-
-from pathlib import Path
-from contextlib import contextmanager
-
-#############################################################################################
-
-DATA_DIR = Path(os.getcwd())
-
-try:
-    DATA_DIR_STAT           = DATA_DIR.stat()
-    DATA_DIR_UID            = DATA_DIR_STAT.st_uid
-    DATA_DIR_GID            = DATA_DIR_STAT.st_gid
-except PermissionError:
-    DATA_DIR_UID            = 0
-    DATA_DIR_GID            = 0
-
-DEFAULT_PUID            = 911
-DEFAULT_PGID            = 911
-RUNNING_AS_UID          = os.getuid()
-RUNNING_AS_GID          = os.getgid()
-EUID                    = os.geteuid()
-EGID                    = os.getegid()
-SUDO_UID                = int(os.environ.get('SUDO_UID', 0))
-SUDO_GID                = int(os.environ.get('SUDO_GID', 0))
-USER: str               = Path('~').expanduser().resolve().name
-
-IS_ROOT = RUNNING_AS_UID == 0
-IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE', 'yes')
-
-FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
-FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
-if RUNNING_AS_UID == 0:
-    try:
-        # if we are running as root it's really hard to figure out what the correct archivebox user should be
-        # as a last resort instead of setting DATA_DIR ownership to 0:0 (which breaks it for non-root users)
-        # check if 911:911 archivebox user exists on host system, and use it instead of 0
-        import pwd
-        if pwd.getpwuid(DEFAULT_PUID).pw_name == 'archivebox':
-            FALLBACK_UID = DEFAULT_PUID
-            FALLBACK_GID = DEFAULT_PGID
-    except Exception:
-        pass
-
-
-os.environ.setdefault('PUID', str(DATA_DIR_UID or EUID or RUNNING_AS_UID or FALLBACK_UID))
-os.environ.setdefault('PGID', str(DATA_DIR_GID or EGID or RUNNING_AS_GID or FALLBACK_GID))
-
-ARCHIVEBOX_USER = int(os.environ['PUID'])
-ARCHIVEBOX_GROUP = int(os.environ['PGID'])
-if not USER:
-    try:
-        # alternative method 1 to get username
-        USER = pwd.getpwuid(ARCHIVEBOX_USER).pw_name
-    except Exception:
-        pass
-        
-if not USER:
-    try:
-        # alternative method 2 to get username
-        import getpass
-        USER = getpass.getuser()
-    except Exception:
-        pass
-    
-if not USER:
-    try:
-        # alternative method 3 to get username
-        USER = os.getlogin() or 'archivebox'
-    except Exception:
-        USER = 'archivebox'
-        
-ARCHIVEBOX_USER_EXISTS = False
-try:
-    pwd.getpwuid(ARCHIVEBOX_USER)
-    ARCHIVEBOX_USER_EXISTS = True
-except Exception:
-    ARCHIVEBOX_USER_EXISTS = False
-    
-
-#############################################################################################
-
-def drop_privileges():
-    """If running as root, drop privileges to the user that owns the data dir (or PUID)"""
-    
-    # always run archivebox as the user that owns the data dir, never as root
-    if os.getuid() == 0:
-        # drop permissions to the user that owns the data dir / provided PUID
-        if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
-            # drop our effective UID to the archivebox user's UID
-            os.seteuid(ARCHIVEBOX_USER)
-            
-            # update environment variables so that subprocesses dont try to write to /root
-            pw_record = pwd.getpwuid(ARCHIVEBOX_USER)
-            os.environ['HOME']     = pw_record.pw_dir
-            os.environ['LOGNAME']  = pw_record.pw_name
-            os.environ['USER']     = pw_record.pw_name
-
-    if ARCHIVEBOX_USER == 0 or not ARCHIVEBOX_USER_EXISTS:
-        print('[yellow]:warning:  Running as [red]root[/red] is not recommended and may make your [blue]DATA_DIR[/blue] inaccessible to other users on your system.[/yellow] (use [blue]sudo[/blue] instead)', file=sys.stderr)
-
-
-@contextmanager
-def SudoPermission(uid=0, fallback=False):
-    """Attempt to run code with sudo permissions for a given user (or root)"""
-    
-    if os.geteuid() == uid:
-        # no need to change effective UID, we are already that user
-        yield
-        return
-
-    try:
-        # change our effective UID to the given UID
-        os.seteuid(uid)
-    except PermissionError as err:
-        if not fallback:
-            raise PermissionError(f'Not enough permissions to run code as uid={uid}, please retry with sudo') from err
-    try:
-        # yield back to the caller so they can run code inside context as root
-        yield
-    finally:
-        # then set effective UID back to DATA_DIR owner
-        try:
-            os.seteuid(ARCHIVEBOX_USER)
-        except PermissionError as err:
-            if not fallback:
-                raise PermissionError(f'Failed to revert uid={uid} back to {ARCHIVEBOX_USER} after running code with sudo') from err
@@ -1,125 +0,0 @@
-__package__ = 'archivebox.config'
-
-import os
-import importlib.metadata
-
-from pathlib import Path
-from functools import cache
-from datetime import datetime
-from typing import Optional
-
-#############################################################################################
-
-IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE', 'yes')
-
-PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent    # archivebox source code dir
-DATA_DIR: Path = Path(os.getcwd()).resolve()                  # archivebox user data dir
-ARCHIVE_DIR: Path = DATA_DIR / 'archive'                      # archivebox snapshot data dir
-
-#############################################################################################
-
-
-@cache
-def detect_installed_version(PACKAGE_DIR: Path=PACKAGE_DIR):
-    """Autodetect the installed archivebox version by using pip package metadata, pyproject.toml file, or package.json file"""
-    try:
-        # if in production install, use pip-installed package metadata
-        return importlib.metadata.version('archivebox').strip()
-    except importlib.metadata.PackageNotFoundError:
-        pass
-
-    try:
-        # if in dev Git repo dir, use pyproject.toml file
-        pyproject_config = (PACKAGE_DIR.parent / 'pyproject.toml').read_text().split('\n')
-        for line in pyproject_config:
-            if line.startswith('version = '):
-                return line.split(' = ', 1)[-1].strip('"').strip()
-    except FileNotFoundError:
-        # building docs, pyproject.toml is not available
-        pass
-
-    # raise Exception('Failed to detect installed archivebox version!')
-    return 'dev'
-
-
-@cache
-def get_COMMIT_HASH() -> Optional[str]:
-    try:
-        git_dir = PACKAGE_DIR.parent / '.git'
-        ref = (git_dir / 'HEAD').read_text().strip().split(' ')[-1]
-        commit_hash = git_dir.joinpath(ref).read_text().strip()
-        return commit_hash
-    except Exception:
-        pass
-
-    try:
-        return list((PACKAGE_DIR.parent / '.git/refs/heads/').glob('*'))[0].read_text().strip()
-    except Exception:
-        pass
-    
-    return None
-    
-@cache
-def get_BUILD_TIME() -> str:
-    if IN_DOCKER:
-        try:
-            # if we're in the archivebox official docker image, /VERSION.txt will contain the build time
-            docker_build_end_time = Path('/VERSION.txt').read_text().rsplit('BUILD_END_TIME=')[-1].split('\n', 1)[0]
-            return docker_build_end_time
-        except Exception:
-            pass
-
-    src_last_modified_unix_timestamp = (PACKAGE_DIR / 'README.md').stat().st_mtime
-    return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime('%Y-%m-%d %H:%M:%S %s')
-
-
-# def get_versions_available_on_github(config):
-#     """
-#     returns a dictionary containing the ArchiveBox GitHub release info for
-#     the recommended upgrade version and the currently installed version
-#     """
-    
-#     # we only want to perform the (relatively expensive) check for new versions
-#     # when its most relevant, e.g. when the user runs a long-running command
-#     subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help'
-#     long_running_commands = ('add', 'schedule', 'update', 'status', 'server')
-#     if subcommand_run_by_user not in long_running_commands:
-#         return None
-    
-#     github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases"
-#     response = requests.get(github_releases_api)
-#     if response.status_code != 200:
-#         stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config)
-#         return None
-#     all_releases = response.json()
-
-#     installed_version = parse_version_string(config['VERSION'])
-
-#     # find current version or nearest older version (to link to)
-#     current_version = None
-#     for idx, release in enumerate(all_releases):
-#         release_version = parse_version_string(release['tag_name'])
-#         if release_version <= installed_version:
-#             current_version = release
-#             break
-
-#     current_version = current_version or all_releases[-1]
-    
-#     # recommended version is whatever comes after current_version in the release list
-#     # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest)
-#     try:
-#         recommended_version = all_releases[idx+1]
-#     except IndexError:
-#         recommended_version = None
-
-#     return {'recommended_version': recommended_version, 'current_version': current_version}
-
-# def can_upgrade(config):
-#     if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']:
-#         recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name'])
-#         current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name'])
-#         return recommended_version > current_version
-#     return False
-
-
-VERSION: str = detect_installed_version()
@@ -1,499 +0,0 @@
-__package__ = 'abx.archivebox'
-
-import os
-import inspect
-from pathlib import Path
-from typing import Any, List, Dict, cast
-from benedict import benedict
-
-from django.http import HttpRequest
-from django.conf import settings
-from django.utils import timezone
-from django.utils.html import format_html, mark_safe
-
-from admin_data_views.typing import TableContext, ItemContext
-from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
-
-import abx
-import archivebox
-from archivebox.config import CONSTANTS
-from archivebox.misc.util import parse_date
-
-from machine.models import InstalledBinary
-
-
-def obj_to_yaml(obj: Any, indent: int=0) -> str:
-    indent_str = "  " * indent
-    if indent == 0:
-        indent_str = '\n'  # put extra newline between top-level entries
-    
-    if isinstance(obj, dict):
-        if not obj:
-            return "{}"
-        result = "\n"
-        for key, value in obj.items():
-            result += f"{indent_str}{key}:{obj_to_yaml(value, indent + 1)}\n"
-        return result
-    
-    elif isinstance(obj, list):
-        if not obj:
-            return "[]"
-        result = "\n"
-        for item in obj:
-            result += f"{indent_str}- {obj_to_yaml(item, indent + 1).lstrip()}\n"
-        return result.rstrip()
-    
-    elif isinstance(obj, str):
-        if "\n" in obj:
-            return f" |\n{indent_str}  " + obj.replace("\n", f"\n{indent_str}  ")
-        else:
-            return f" {obj}"
-    
-    elif isinstance(obj, (int, float, bool)):
-        return f" {str(obj)}"
-    
-    elif callable(obj):
-        source = '\n'.join(
-            '' if 'def ' in line else line
-            for line in inspect.getsource(obj).split('\n')
-            if line.strip()
-        ).split('lambda: ')[-1].rstrip(',')
-        return f" {indent_str}  " + source.replace("\n", f"\n{indent_str}  ")
-    
-    else:
-        return f" {str(obj)}"
-
-@render_with_table_view
-def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
-    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
-
-    rows = {
-        "Binary Name": [],
-        "Found Version": [],
-        "From Plugin": [],
-        "Provided By": [],
-        "Found Abspath": [],
-        "Related Configuration": [],
-        # "Overrides": [],
-        # "Description": [],
-    }
-
-    relevant_configs = {
-        key: val
-        for key, val in FLAT_CONFIG.items()
-        if '_BINARY' in key or '_VERSION' in key
-    }
-
-    for plugin_id, plugin in abx.get_all_plugins().items():
-        if not plugin.hooks.get('get_BINARIES'):
-            continue
-        
-        for binary in plugin.hooks.get_BINARIES().values():
-            try:
-                installed_binary = InstalledBinary.objects.get_from_db_or_cache(binary)
-                binary = installed_binary.load_from_db()
-            except Exception as e:
-                print(e)
-
-            rows['Binary Name'].append(ItemLink(binary.name, key=binary.name))
-            rows['Found Version'].append(f'✅ {binary.loaded_version}' if binary.loaded_version else '❌ missing')
-            rows['From Plugin'].append(plugin.package)
-            rows['Provided By'].append(
-                ', '.join(
-                    f'[{binprovider.name}]' if binprovider.name == getattr(binary.loaded_binprovider, 'name', None) else binprovider.name
-                    for binprovider in binary.binproviders_supported
-                    if binprovider
-                )
-                # binary.loaded_binprovider.name
-                # if binary.loaded_binprovider else
-                # ', '.join(getattr(provider, 'name', str(provider)) for provider in binary.binproviders_supported)
-            )
-            rows['Found Abspath'].append(str(binary.loaded_abspath or '❌ missing'))
-            rows['Related Configuration'].append(mark_safe(', '.join(
-                f'<a href="/admin/environment/config/{config_key}/">{config_key}</a>'
-                for config_key, config_value in relevant_configs.items()
-                    if str(binary.name).lower().replace('-', '').replace('_', '').replace('ytdlp', 'youtubedl') in config_key.lower()
-                    or config_value.lower().endswith(binary.name.lower())
-                    # or binary.name.lower().replace('-', '').replace('_', '') in str(config_value).lower()
-            )))
-            # if not binary.overrides:
-                # import ipdb; ipdb.set_trace()
-            # rows['Overrides'].append(str(obj_to_yaml(binary.overrides) or str(binary.overrides))[:200])
-            # rows['Description'].append(binary.description)
-
-    return TableContext(
-        title="Binaries",
-        table=rows,
-    )
-
-@render_with_item_view
-def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-
-    assert request.user and request.user.is_superuser, 'Must be a superuser to view configuration settings.'
-
-    binary = None
-    plugin = None
-    for plugin_id, plugin in abx.get_all_plugins().items():
-        try:
-            for loaded_binary in plugin['hooks'].get_BINARIES().values():
-                if loaded_binary.name == key:
-                    binary = loaded_binary
-                    plugin = plugin
-                    # break  # last write wins
-        except Exception as e:
-            print(e)
-
-    assert plugin and binary, f'Could not find a binary matching the specified name: {key}'
-
-    try:
-        binary = binary.load()
-    except Exception as e:
-        print(e)
-
-    return ItemContext(
-        slug=key,
-        title=key,
-        data=[
-            {
-                "name": binary.name,
-                "description": binary.abspath,
-                "fields": {
-                    'plugin': plugin['package'],
-                    'binprovider': binary.loaded_binprovider,
-                    'abspath': binary.loaded_abspath,
-                    'version': binary.loaded_version,
-                    'overrides': obj_to_yaml(binary.overrides),
-                    'providers': obj_to_yaml(binary.binproviders_supported),
-                },
-                "help_texts": {
-                    # TODO
-                },
-            },
-        ],
-    )
-
-
-@render_with_table_view
-def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
-
-    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
-
-    rows = {
-        "Label": [],
-        "Version": [],
-        "Author": [],
-        "Package": [],
-        "Source Code": [],
-        "Config": [],
-        "Binaries": [],
-        "Package Managers": [],
-        # "Search Backends": [],
-    }
-
-    config_colors = {
-        '_BINARY': '#339',
-        'USE_': 'green',
-        'SAVE_': 'green',
-        '_ARGS': '#33e',
-        'KEY': 'red',
-        'COOKIES': 'red',
-        'AUTH': 'red',
-        'SECRET': 'red',
-        'TOKEN': 'red',
-        'PASSWORD': 'red',
-        'TIMEOUT': '#533',
-        'RETRIES': '#533',
-        'MAX': '#533',
-        'MIN': '#533',
-    }
-    def get_color(key):
-        for pattern, color in config_colors.items():
-            if pattern in key:
-                return color
-        return 'black'
-
-    for plugin_id, plugin in abx.get_all_plugins().items():
-        plugin.hooks.get_BINPROVIDERS = plugin.hooks.get('get_BINPROVIDERS', lambda: {})
-        plugin.hooks.get_BINARIES = plugin.hooks.get('get_BINARIES', lambda: {})
-        plugin.hooks.get_CONFIG = plugin.hooks.get('get_CONFIG', lambda: {})
-        
-        rows['Label'].append(ItemLink(plugin.label, key=plugin.package))
-        rows['Version'].append(str(plugin.version))
-        rows['Author'].append(mark_safe(f'<a href="{plugin.homepage}" target="_blank">{plugin.author}</a>'))
-        rows['Package'].append(ItemLink(plugin.package, key=plugin.package))
-        rows['Source Code'].append(format_html('<code>{}</code>', str(plugin.source_code).replace(str(Path('~').expanduser()), '~')))
-        rows['Config'].append(mark_safe(''.join(
-            f'<a href="/admin/environment/config/{key}/"><b><code style="color: {get_color(key)};">{key}</code></b>=<code>{value}</code></a><br/>'
-            for configdict in plugin.hooks.get_CONFIG().values()
-                for key, value in benedict(configdict).items()
-        )))
-        rows['Binaries'].append(mark_safe(', '.join(
-            f'<a href="/admin/environment/binaries/{binary.name}/"><code>{binary.name}</code></a>'
-            for binary in plugin.hooks.get_BINARIES().values()
-        )))
-        rows['Package Managers'].append(mark_safe(', '.join(
-            f'<a href="/admin/environment/binproviders/{binprovider.name}/"><code>{binprovider.name}</code></a>'
-            for binprovider in plugin.hooks.get_BINPROVIDERS().values()
-        )))
-        # rows['Search Backends'].append(mark_safe(', '.join(
-        #     f'<a href="/admin/environment/searchbackends/{searchbackend.name}/"><code>{searchbackend.name}</code></a>'
-        #     for searchbackend in plugin.SEARCHBACKENDS.values()
-        # )))
-
-    return TableContext(
-        title="Installed plugins",
-        table=rows,
-    )
-
-@render_with_item_view
-def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-
-    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
-
-    plugin_id = None
-    for check_plugin_id, loaded_plugin in settings.PLUGINS.items():
-        if check_plugin_id.split('.')[-1] == key.split('.')[-1]:
-            plugin_id = check_plugin_id
-            break
-
-    assert plugin_id, f'Could not find a plugin matching the specified name: {key}'
-
-    plugin = abx.get_plugin(plugin_id)
-
-    return ItemContext(
-        slug=key,
-        title=key,
-        data=[
-            {
-                "name": plugin.package,
-                "description": plugin.label,
-                "fields": {
-                    "id": plugin.id,
-                    "package": plugin.package,
-                    "label": plugin.label,
-                    "version": plugin.version,
-                    "author": plugin.author,
-                    "homepage": plugin.homepage,
-                    "dependencies": getattr(plugin, 'DEPENDENCIES', []),
-                    "source_code": plugin.source_code,
-                    "hooks": plugin.hooks,
-                },
-                "help_texts": {
-                    # TODO
-                },
-            },
-        ],
-    )
-
-
-@render_with_table_view
-def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
-
-    rows = {
-        "Name": [],
-        "State": [],
-        "PID": [],
-        "Started": [],
-        "Command": [],
-        "Logfile": [],
-        "Exit Status": [],
-    }
-    
-    from queues.supervisor_util import get_existing_supervisord_process
-    
-    supervisor = get_existing_supervisord_process()
-    if supervisor is None:
-        return TableContext(
-            title="No running worker processes",
-            table=rows,
-        )
-        
-    all_config_entries = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or [])
-    all_config = {config["name"]: benedict(config) for config in all_config_entries}
-
-    # Add top row for supervisord process manager
-    rows["Name"].append(ItemLink('supervisord', key='supervisord'))
-    rows["State"].append(supervisor.getState()['statename'])
-    rows['PID'].append(str(supervisor.getPID()))
-    rows["Started"].append('-')
-    rows["Command"].append('supervisord --configuration=tmp/supervisord.conf')
-    rows["Logfile"].append(
-        format_html(
-            '<a href="/admin/environment/logs/{}/">{}</a>',
-            'supervisord',
-            'logs/supervisord.log',
-        )
-    )
-    rows['Exit Status'].append('0')
-
-    # Add a row for each worker process managed by supervisord
-    for proc in cast(List[Dict[str, Any]], supervisor.getAllProcessInfo()):
-        proc = benedict(proc)
-        # {
-        #     "name": "daphne",
-        #     "group": "daphne",
-        #     "start": 1725933056,
-        #     "stop": 0,
-        #     "now": 1725933438,
-        #     "state": 20,
-        #     "statename": "RUNNING",
-        #     "spawnerr": "",
-        #     "exitstatus": 0,
-        #     "logfile": "logs/server.log",
-        #     "stdout_logfile": "logs/server.log",
-        #     "stderr_logfile": "",
-        #     "pid": 33283,
-        #     "description": "pid 33283, uptime 0:06:22",
-        # }
-        rows["Name"].append(ItemLink(proc.name, key=proc.name))
-        rows["State"].append(proc.statename)
-        rows['PID'].append(proc.description.replace('pid ', ''))
-        rows["Started"].append(parse_date(proc.start).strftime("%Y-%m-%d %H:%M:%S") if proc.start else '')
-        rows["Command"].append(all_config[proc.name].command)
-        rows["Logfile"].append(
-            format_html(
-                '<a href="/admin/environment/logs/{}/">{}</a>',
-                proc.stdout_logfile.split("/")[-1].split('.')[0],
-                proc.stdout_logfile,
-            )
-        )
-        rows["Exit Status"].append(str(proc.exitstatus))
-
-    return TableContext(
-        title="Running worker processes",
-        table=rows,
-    )
-
-
-@render_with_item_view
-def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
-
-    from queues.supervisor_util import get_existing_supervisord_process, get_worker
-    from queues.settings import SUPERVISORD_CONFIG_FILE
-
-    supervisor = get_existing_supervisord_process()
-    if supervisor is None:
-        return ItemContext(
-            slug='none',
-            title='error: No running supervisord process.',
-            data=[],
-        )
-
-    all_config = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or [])
-
-    if key == 'supervisord':
-        relevant_config = SUPERVISORD_CONFIG_FILE.read_text()
-        relevant_logs = cast(str, supervisor.readLog(0, 10_000_000))
-        start_ts = [line for line in relevant_logs.split("\n") if "RPC interface 'supervisor' initialized" in line][-1].split(",", 1)[0]
-        uptime = str(timezone.now() - parse_date(start_ts)).split(".")[0]
-
-        proc = benedict(
-            {
-                "name": "supervisord",
-                "pid": supervisor.getPID(),
-                "statename": supervisor.getState()["statename"],
-                "start": start_ts,
-                "stop": None,
-                "exitstatus": "",
-                "stdout_logfile": "logs/supervisord.log",
-                "description": f'pid 000, uptime {uptime}',
-            }
-        )
-    else:
-        proc = benedict(get_worker(supervisor, key) or {})
-        relevant_config = [config for config in all_config if config['name'] == key][0]
-        relevant_logs = supervisor.tailProcessStdoutLog(key, 0, 10_000_000)[0]
-
-    return ItemContext(
-        slug=key,
-        title=key,
-        data=[
-            {
-                "name": key,
-                "description": key,
-                "fields": {
-                    "Command": proc.name,
-                    "PID": proc.pid,
-                    "State": proc.statename,
-                    "Started": parse_date(proc.start).strftime("%Y-%m-%d %H:%M:%S") if proc.start else "",
-                    "Stopped": parse_date(proc.stop).strftime("%Y-%m-%d %H:%M:%S") if proc.stop else "",
-                    "Exit Status": str(proc.exitstatus),
-                    "Logfile": proc.stdout_logfile,
-                    "Uptime": (proc.description or "").split("uptime ", 1)[-1],
-                    "Config": relevant_config,
-                    "Logs": relevant_logs,
-                },
-                "help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"},
-            },
-        ],
-    )
-
-
-@render_with_table_view
-def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
-
-
-    log_files = CONSTANTS.LOGS_DIR.glob("*.log")
-    log_files = sorted(log_files, key=os.path.getmtime)[::-1]
-
-    rows = {
-        "Name": [],
-        "Last Updated": [],
-        "Size": [],
-        "Most Recent Lines": [],
-    }
-
-    # Add a row for each worker process managed by supervisord
-    for logfile in log_files:
-        st = logfile.stat()
-        rows["Name"].append(ItemLink("logs" + str(logfile).rsplit("/logs", 1)[-1], key=logfile.name))
-        rows["Last Updated"].append(parse_date(st.st_mtime).strftime("%Y-%m-%d %H:%M:%S"))
-        rows["Size"].append(f'{st.st_size//1000} kb')
-
-        with open(logfile, 'rb') as f:
-            try:
-                f.seek(-1024, os.SEEK_END)
-            except OSError:
-                f.seek(0)
-            last_lines = f.read().decode('utf-8', errors='replace').split("\n")
-            non_empty_lines = [line for line in last_lines if line.strip()]
-            rows["Most Recent Lines"].append(non_empty_lines[-1])
-
-    return TableContext(
-        title="Debug Log files",
-        table=rows,
-    )
-
-
-@render_with_item_view
-def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
-
-    from django.conf import settings
-    
-    log_file = [logfile for logfile in CONSTANTS.LOGS_DIR.glob('*.log') if key in logfile.name][0]
-
-    log_text = log_file.read_text()
-    log_stat = log_file.stat()
-
-    return ItemContext(
-        slug=key,
-        title=key,
-        data=[
-            {
-                "name": key,
-                "description": key,
-                "fields": {
-                    "Path": str(log_file),
-                    "Size": f"{log_stat.st_size//1000} kb",
-                    "Last Updated": parse_date(log_stat.st_mtime).strftime("%Y-%m-%d %H:%M:%S"),
-                    "Tail": "\n".join(log_text[-10_000:].split("\n")[-20:]),
-                    "Full Log": log_text,
-                },
-            },
-        ],
-    )
@@ -0,0 +1,119 @@
+from pathlib import Path
+from typing import Optional, Dict, Union, Tuple, Callable, Pattern, Type, Any, List
+from mypy_extensions import TypedDict
+
+
+
+SimpleConfigValue = Union[str, bool, int, None, Pattern, Dict[str, Any]]
+SimpleConfigValueDict = Dict[str, SimpleConfigValue]
+SimpleConfigValueGetter = Callable[[], SimpleConfigValue]
+ConfigValue = Union[SimpleConfigValue, SimpleConfigValueDict, SimpleConfigValueGetter]
+
+class AttrDict(dict):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.__dict__ = self
+
+class BaseConfig(TypedDict):
+    pass
+
+class ConfigDict(BaseConfig, AttrDict, total=False):
+    """
+    # Regenerate by pasting this quine into `archivebox shell` 🥚
+    from archivebox.config import ConfigDict, CONFIG_DEFAULTS
+    print('class ConfigDict(BaseConfig, total=False):')
+    print('    ' + '"'*3 + ConfigDict.__doc__ + '"'*3)
+    for section, configs in CONFIG_DEFAULTS.items():
+        for key, attrs in configs.items():
+            Type, default = attrs['type'], attrs['default']
+            if default is None:
+                print(f'    {key}: Optional[{Type.__name__}]')
+            else:
+                print(f'    {key}: {Type.__name__}')
+        print()
+    """
+
+    IS_TTY: bool
+    USE_COLOR: bool
+    SHOW_PROGRESS: bool
+    IN_DOCKER: bool
+
+    PACKAGE_DIR: Path
+    OUTPUT_DIR: Path
+    CONFIG_FILE: Path
+    ONLY_NEW: bool
+    TIMEOUT: int
+    MEDIA_TIMEOUT: int
+    OUTPUT_PERMISSIONS: str
+    RESTRICT_FILE_NAMES: str
+    URL_DENYLIST: str
+
+    SECRET_KEY: Optional[str]
+    BIND_ADDR: str
+    ALLOWED_HOSTS: str
+    DEBUG: bool
+    PUBLIC_INDEX: bool
+    PUBLIC_SNAPSHOTS: bool
+    FOOTER_INFO: str
+
+    SAVE_TITLE: bool
+    SAVE_FAVICON: bool
+    SAVE_WGET: bool
+    SAVE_WGET_REQUISITES: bool
+    SAVE_SINGLEFILE: bool
+    SAVE_READABILITY: bool
+    SAVE_MERCURY: bool
+    SAVE_PDF: bool
+    SAVE_SCREENSHOT: bool
+    SAVE_DOM: bool
+    SAVE_WARC: bool
+    SAVE_GIT: bool
+    SAVE_MEDIA: bool
+    SAVE_ARCHIVE_DOT_ORG: bool
+
+    RESOLUTION: str
+    GIT_DOMAINS: str
+    CHECK_SSL_VALIDITY: bool
+    CURL_USER_AGENT: str
+    WGET_USER_AGENT: str
+    CHROME_USER_AGENT: str
+    COOKIES_FILE: Union[str, Path, None]
+    CHROME_USER_DATA_DIR: Union[str, Path, None]
+    CHROME_TIMEOUT: int
+    CHROME_HEADLESS: bool
+    CHROME_SANDBOX: bool
+
+    USE_CURL: bool
+    USE_WGET: bool
+    USE_SINGLEFILE: bool
+    USE_READABILITY: bool
+    USE_MERCURY: bool
+    USE_GIT: bool
+    USE_CHROME: bool
+    USE_YOUTUBEDL: bool
+    CURL_BINARY: str
+    GIT_BINARY: str
+    WGET_BINARY: str
+    SINGLEFILE_BINARY: str
+    READABILITY_BINARY: str
+    MERCURY_BINARY: str
+    YOUTUBEDL_BINARY: str
+    CHROME_BINARY: Optional[str]
+
+    YOUTUBEDL_ARGS: List[str]
+    WGET_ARGS: List[str]
+    CURL_ARGS: List[str]
+    GIT_ARGS: List[str]
+    TAG_SEPARATOR_PATTERN: str
+
+
+ConfigDefaultValueGetter = Callable[[ConfigDict], ConfigValue]
+ConfigDefaultValue = Union[ConfigValue, ConfigDefaultValueGetter]
+
+ConfigDefault = TypedDict('ConfigDefault', {
+    'default': ConfigDefaultValue,
+    'type': Optional[Type],
+    'aliases': Optional[Tuple[str, ...]],
+}, total=False)
+
+ConfigDefaultDict = Dict[str, ConfigDefault]
@@ -1,31 +1,2 @@
 __package__ = 'archivebox.core'

-import abx
-
-@abx.hookimpl
-def register_admin(admin_site):
-    """Register the core.models views (Snapshot, ArchiveResult, Tag, etc.) with the admin site"""
-    from core.admin import register_admin
-    register_admin(admin_site)
-
-
-
-@abx.hookimpl
-def get_CONFIG():
-    from archivebox.config.common import (
-        SHELL_CONFIG,
-        STORAGE_CONFIG,
-        GENERAL_CONFIG,
-        SERVER_CONFIG,
-        ARCHIVING_CONFIG,
-        SEARCH_BACKEND_CONFIG,
-    )
-    return {
-        'SHELL_CONFIG': SHELL_CONFIG,
-        'STORAGE_CONFIG': STORAGE_CONFIG,
-        'GENERAL_CONFIG': GENERAL_CONFIG,
-        'SERVER_CONFIG': SERVER_CONFIG,
-        'ARCHIVING_CONFIG': ARCHIVING_CONFIG,
-        'SEARCHBACKEND_CONFIG': SEARCH_BACKEND_CONFIG,
-    }
-
@@ -1,41 +0,0 @@
-__package__ = 'archivebox.core'
-
-from typing import ClassVar
-
-from statemachine import State
-
-from core.models import Snapshot, ArchiveResult
-from core.statemachines import SnapshotMachine, ArchiveResultMachine
-from actors.actor import ActorType
-
-
-class SnapshotActor(ActorType[Snapshot]):
-    Model = Snapshot
-    StateMachineClass = SnapshotMachine
-    
-    ACTIVE_STATE: ClassVar[State] = SnapshotMachine.started
-    FINAL_STATES: ClassVar[list[State]] = SnapshotMachine.final_states
-    STATE_FIELD_NAME: ClassVar[str] = SnapshotMachine.state_field_name
-    
-    MAX_CONCURRENT_ACTORS: ClassVar[int] = 3
-    MAX_TICK_TIME: ClassVar[int] = 10
-    CLAIM_FROM_TOP_N: ClassVar[int] = MAX_CONCURRENT_ACTORS * 10
-
-
-
-class ArchiveResultActor(ActorType[ArchiveResult]):
-    Model = ArchiveResult
-    StateMachineClass = ArchiveResultMachine
-    
-    ACTIVE_STATE: ClassVar[State] = ArchiveResultMachine.started
-    FINAL_STATES: ClassVar[list[State]] = ArchiveResultMachine.final_states
-    STATE_FIELD_NAME: ClassVar[str] = ArchiveResultMachine.state_field_name
-    
-    MAX_CONCURRENT_ACTORS: ClassVar[int] = 6
-    MAX_TICK_TIME: ClassVar[int] = 60
-    CLAIM_FROM_TOP_N: ClassVar[int] = MAX_CONCURRENT_ACTORS * 10
-
-    # @classproperty
-    # def qs(cls) -> QuerySet[ModelType]:
-    #     """Get the unfiltered and unsorted QuerySet of all objects that this Actor might care about."""
-    #     return cls.Model.objects.filter(extractor='favicon')
@@ -1,20 +1,754 @@
 __package__ = 'archivebox.core'

+import os
+import json
+
+from io import StringIO
+from pathlib import Path
+from contextlib import redirect_stdout
+from datetime import datetime, timezone
+from typing import Dict, Any
+
+from django.contrib import admin
+from django.db.models import Count, Q, Prefetch
+from django.urls import path, reverse, resolve
+from django.utils import timezone
+from django.utils.functional import cached_property
+from django.utils.html import format_html
+from django.utils.safestring import mark_safe
+from django.shortcuts import render, redirect
 from django.contrib.auth import get_user_model
+from django.core.paginator import Paginator
+from django.core.exceptions import ValidationError
+from django.conf import settings
+from django import forms


-from core.models import Snapshot, ArchiveResult, Tag
-from core.admin_tags import TagAdmin
-from core.admin_snapshots import SnapshotAdmin
-from core.admin_archiveresults import ArchiveResultAdmin
-from core.admin_users import UserAdmin
+from signal_webhooks.admin import WebhookAdmin
+from signal_webhooks.utils import get_webhook_model
+# from plugantic.admin import CustomPlugin

-import abx
+from ..util import htmldecode, urldecode, ansi_to_html
+
+from core.models import Snapshot, ArchiveResult, Tag, SnapshotTag
+from core.forms import AddLinkForm
+from core.mixins import SearchResultsAdminMixin
+from api.models import APIToken
+from abid_utils.models import get_or_create_system_user_pk
+from abid_utils.admin import ABIDModelAdmin
+
+from index.html import snapshot_icons
+from logging_util import printable_filesize
+from main import add, remove
+from extractors import archive_links


-@abx.hookimpl
-def register_admin(admin_site):
-    admin_site.register(get_user_model(), UserAdmin)
-    admin_site.register(ArchiveResult, ArchiveResultAdmin)
-    admin_site.register(Snapshot, SnapshotAdmin)
-    admin_site.register(Tag, TagAdmin)
+CONFIG = settings.CONFIG
+
+GLOBAL_CONTEXT = {'VERSION': CONFIG.VERSION, 'VERSIONS_AVAILABLE': CONFIG.VERSIONS_AVAILABLE, 'CAN_UPGRADE': CONFIG.CAN_UPGRADE}
+
+# Admin URLs
+# /admin/
+# /admin/login/
+# /admin/core/
+# /admin/core/snapshot/
+# /admin/core/snapshot/:uuid/
+# /admin/core/tag/
+# /admin/core/tag/:uuid/
+
+
+# TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
+
+
+class ArchiveBoxAdmin(admin.AdminSite):
+    site_header = 'ArchiveBox'
+    index_title = 'Links'
+    site_title = 'Index'
+    namespace = 'admin'
+
+    def get_urls(self):
+        return [
+            path('core/snapshot/add/', self.add_view, name='Add'),
+        ] + super().get_urls()
+
+    def add_view(self, request):
+        if not request.user.is_authenticated:
+            return redirect(f'/admin/login/?next={request.path}')
+
+        request.current_app = self.name
+        context: Dict[str, Any] = {
+            **self.each_context(request),
+            'title': 'Add URLs',
+        }
+
+        if request.method == 'GET':
+            context['form'] = AddLinkForm()
+
+        elif request.method == 'POST':
+            form = AddLinkForm(request.POST)
+            if form.is_valid():
+                url = form.cleaned_data["url"]
+                print(f'[+] Adding URL: {url}')
+                depth = 0 if form.cleaned_data["depth"] == "0" else 1
+                input_kwargs = {
+                    "urls": url,
+                    "depth": depth,
+                    "update_all": False,
+                    "out_dir": CONFIG.OUTPUT_DIR,
+                }
+                add_stdout = StringIO()
+                with redirect_stdout(add_stdout):
+                   add(**input_kwargs)
+                print(add_stdout.getvalue())
+
+                context.update({
+                    "stdout": ansi_to_html(add_stdout.getvalue().strip()),
+                    "form": AddLinkForm(),
+                })
+            else:
+                context["form"] = form
+
+        return render(template_name='add.html', request=request, context=context)
+
+
+archivebox_admin = ArchiveBoxAdmin()
+archivebox_admin.register(get_user_model())
+archivebox_admin.disable_action('delete_selected')
+
+# archivebox_admin.register(CustomPlugin)
+
+# patch admin with methods to add data views (implemented by admin_data_views package)
+# https://github.com/MrThearMan/django-admin-data-views
+# https://mrthearman.github.io/django-admin-data-views/setup/
+############### Additional sections are defined in settings.ADMIN_DATA_VIEWS #########
+from admin_data_views.admin import get_app_list, admin_data_index_view, get_admin_data_urls, get_urls
+
+archivebox_admin.get_app_list = get_app_list.__get__(archivebox_admin, ArchiveBoxAdmin)
+archivebox_admin.admin_data_index_view = admin_data_index_view.__get__(archivebox_admin, ArchiveBoxAdmin)       # type: ignore
+archivebox_admin.get_admin_data_urls = get_admin_data_urls.__get__(archivebox_admin, ArchiveBoxAdmin)           # type: ignore
+archivebox_admin.get_urls = get_urls(archivebox_admin.get_urls).__get__(archivebox_admin, ArchiveBoxAdmin)
+
+
+class AccelleratedPaginator(Paginator):
+    """
+    Accellerated Pagniator ignores DISTINCT when counting total number of rows.
+    Speeds up SELECT Count(*) on Admin views by >20x.
+    https://hakibenita.com/optimizing-the-django-admin-paginator
+    """
+
+    @cached_property
+    def count(self):
+        if self.object_list._has_filters():
+            # fallback to normal count method on filtered queryset
+            return super().count
+        else:
+            # otherwise count total rows in a separate fast query
+            return self.object_list.model.objects.count()
+    
+        # Alternative approach for PostgreSQL: fallback count takes > 200ms
+        # from django.db import connection, transaction, OperationalError
+        # with transaction.atomic(), connection.cursor() as cursor:
+        #     cursor.execute('SET LOCAL statement_timeout TO 200;')
+        #     try:
+        #         return super().count
+        #     except OperationalError:
+        #         return 9999999999999
+
+
+class ArchiveResultInline(admin.TabularInline):
+    name = 'Archive Results Log'
+    model = ArchiveResult
+    parent_model = Snapshot
+    # fk_name = 'snapshot'
+    extra = 0
+    sort_fields = ('end_ts', 'extractor', 'output', 'status', 'cmd_version')
+    readonly_fields = ('result_id', 'completed', 'extractor', 'command', 'version')
+    fields = ('id', 'start_ts', 'end_ts', *readonly_fields, 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
+    # exclude = ('id',)
+    ordering = ('end_ts',)
+    show_change_link = True
+    # # classes = ['collapse']
+    # # list_display_links = ['abid']
+
+    def get_parent_object_from_request(self, request):
+        resolved = resolve(request.path_info)
+        return self.parent_model.objects.get(pk=resolved.kwargs['object_id'])
+
+    @admin.display(
+        description='Completed',
+        ordering='end_ts',
+    )
+    def completed(self, obj):
+        return format_html('<p style="white-space: nowrap">{}</p>', obj.end_ts.strftime('%Y-%m-%d %H:%M:%S'))
+
+    def result_id(self, obj):
+        return format_html('<a href="{}"><code style="font-size: 10px">[{}]</code></a>', reverse('admin:core_archiveresult_change', args=(obj.id,)), obj.abid)
+    
+    def command(self, obj):
+        return format_html('<small><code>{}</code></small>', " ".join(obj.cmd or []))
+    
+    def version(self, obj):
+        return format_html('<small><code>{}</code></small>', obj.cmd_version or '-')
+    
+    def get_formset(self, request, obj=None, **kwargs):
+        formset = super().get_formset(request, obj, **kwargs)
+        snapshot = self.get_parent_object_from_request(request)
+
+        # import ipdb; ipdb.set_trace()
+        formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
+        
+        # default values for new entries
+        formset.form.base_fields['status'].initial = 'succeeded'
+        formset.form.base_fields['start_ts'].initial = timezone.now()
+        formset.form.base_fields['end_ts'].initial = timezone.now()
+        formset.form.base_fields['cmd_version'].initial = '-'
+        formset.form.base_fields['pwd'].initial = str(snapshot.link_dir)
+        formset.form.base_fields['created_by'].initial = request.user
+        formset.form.base_fields['cmd'] = forms.JSONField(initial=['-'])
+        formset.form.base_fields['output'].initial = 'Manually recorded cmd output...'
+        
+        if obj is not None:
+            # hidden values for existing entries and new entries
+            formset.form.base_fields['start_ts'].widget = formset.form.base_fields['start_ts'].hidden_widget()
+            formset.form.base_fields['end_ts'].widget = formset.form.base_fields['end_ts'].hidden_widget()
+            formset.form.base_fields['cmd'].widget = formset.form.base_fields['cmd'].hidden_widget()
+            formset.form.base_fields['pwd'].widget = formset.form.base_fields['pwd'].hidden_widget()
+            formset.form.base_fields['created_by'].widget = formset.form.base_fields['created_by'].hidden_widget()
+            formset.form.base_fields['cmd_version'].widget = formset.form.base_fields['cmd_version'].hidden_widget()
+        return formset
+    
+    def get_readonly_fields(self, request, obj=None):
+        if obj is not None:
+            return self.readonly_fields
+        else:
+            return []
+
+
+class TagInline(admin.TabularInline):
+    model = Tag.snapshot_set.through       # type: ignore
+    # fk_name = 'snapshot'
+    fields = ('id', 'tag')
+    extra = 1
+    # min_num = 1
+    max_num = 1000
+    autocomplete_fields = (
+        'tag',
+    )
+
+from django.contrib.admin.helpers import ActionForm
+from django.contrib.admin.widgets import FilteredSelectMultiple
+
+# class AutocompleteTags:
+#     model = Tag
+#     search_fields = ['name']
+#     name = 'name'
+#     # source_field = 'name'
+#     remote_field = Tag._meta.get_field('name')
+
+# class AutocompleteTagsAdminStub:
+#     name = 'admin'
+
+
+class SnapshotActionForm(ActionForm):
+    tags = forms.ModelMultipleChoiceField(
+        queryset=Tag.objects.all(),
+        required=False,
+        widget=FilteredSelectMultiple(
+            'core_tag__name',
+            False,
+        ),
+    )
+
+    # TODO: allow selecting actions for specific extractors? is this useful?
+    # extractor = forms.ChoiceField(
+    #     choices=ArchiveResult.EXTRACTOR_CHOICES,
+    #     required=False,
+    #     widget=forms.MultileChoiceField(attrs={'class': "form-control"})
+    # )
+
+
+def get_abid_info(self, obj):
+    return format_html(
+        # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+        '''
+        <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
+        <br/><hr/>
+        <div style="opacity: 0.8">
+        &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
+        &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
+        &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/>
+        <hr/>
+        &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; {}</code> &nbsp; &nbsp; &nbsp;&nbsp; {}: <code style="user-select: all">{}</code><br/>
+        &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; "><b style="user-select: all">{}</b> &nbsp; &nbsp; {}</code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <span style="display:inline-block; vertical-align: -4px; width: 290px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}: <code style="user-select: all">{}</code></span>
+        &nbsp; SALT: &nbsp; <code style="font-size: 10px;"><b style="display:inline-block; user-select: all; width: 50px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}</b></code><br/>
+        &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; {}: <code style="user-select: all">{}</code><br/>
+        &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px;"><b style="user-select: all">{}</b> &nbsp; &nbsp; &nbsp; {}</code> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;  {}: <code style="user-select: all">{}</code>
+        <br/><hr/>
+        &nbsp; &nbsp; <small style="opacity: 0.5">.old_id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/>
+        </div>
+        ''',
+        obj.api_url, obj.api_url, obj.api_docs_url,
+        str(obj.abid),
+        str(obj.ABID.uuid),
+        str(obj.id),
+        obj.ABID.ts, str(obj.ABID.uuid)[0:14], obj.abid_ts_src, obj.abid_values['ts'].isoformat() if isinstance(obj.abid_values['ts'], datetime) else obj.abid_values['ts'],
+        obj.ABID.uri, str(obj.ABID.uuid)[14:26], obj.abid_uri_src, str(obj.abid_values['uri']),
+        obj.ABID.uri_salt,
+        obj.ABID.subtype, str(obj.ABID.uuid)[26:28], obj.abid_subtype_src, str(obj.abid_values['subtype']),
+        obj.ABID.rand, str(obj.ABID.uuid)[28:36], obj.abid_rand_src, str(obj.abid_values['rand'])[-7:],
+        str(getattr(obj, 'old_id', '')),
+    )
+
+
+@admin.register(Snapshot, site=archivebox_admin)
+class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
+    list_display = ('added', 'title_str', 'files', 'size', 'url_str')
+    sort_fields = ('title_str', 'url_str', 'added', 'files')
+    readonly_fields = ('tags_str', 'timestamp', 'admin_actions', 'status_info', 'bookmarked', 'added', 'updated', 'created', 'modified', 'API', 'link_dir')
+    search_fields = ('id', 'url', 'abid', 'old_id', 'timestamp', 'title', 'tags__name')
+    list_filter = ('added', 'updated', 'archiveresult__status', 'created_by', 'tags__name')
+    fields = ('url', 'created_by', 'title', *readonly_fields)
+    ordering = ['-added']
+    actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
+    inlines = [TagInline, ArchiveResultInline]
+    list_per_page = min(max(5, CONFIG.SNAPSHOTS_PER_PAGE), 5000)
+
+    action_form = SnapshotActionForm
+    paginator = AccelleratedPaginator
+
+    save_on_top = True
+    show_full_result_count = False
+
+    def changelist_view(self, request, extra_context=None):
+        extra_context = extra_context or {}
+        try:
+            return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)
+        except Exception as e:
+            self.message_user(request, f'Error occurred while loading the page: {str(e)} {request.GET} {request.POST}')
+            return super().changelist_view(request, GLOBAL_CONTEXT)
+
+    def change_view(self, request, object_id, form_url="", extra_context=None):
+        snapshot = None
+
+        try:
+            snapshot = snapshot or Snapshot.objects.get(id=object_id)
+        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
+            pass
+        
+        try:
+            snapshot = snapshot or Snapshot.objects.get(abid=Snapshot.abid_prefix + object_id.split('_', 1)[-1])
+        except (Snapshot.DoesNotExist, ValidationError):
+            pass
+
+
+        try:
+            snapshot = snapshot or Snapshot.objects.get(old_id=object_id)
+        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
+            pass
+
+        if snapshot:
+            object_id = str(snapshot.id)
+
+        return super().change_view(
+            request,
+            object_id,
+            form_url,
+            extra_context=extra_context,
+        )
+
+    def get_urls(self):
+        urls = super().get_urls()
+        custom_urls = [
+            path('grid/', self.admin_site.admin_view(self.grid_view), name='grid')
+        ]
+        return custom_urls + urls
+
+    # def get_queryset(self, request):
+    #     # tags_qs = SnapshotTag.objects.all().select_related('tag')
+    #     # prefetch = Prefetch('snapshottag_set', queryset=tags_qs)
+
+    #     self.request = request
+    #     return super().get_queryset(request).prefetch_related('archiveresult_set').distinct()  # .annotate(archiveresult_count=Count('archiveresult'))
+
+    def tag_list(self, obj):
+        return ', '.join(tag.name for tag in obj.tags.all())
+
+    # TODO: figure out a different way to do this, you cant nest forms so this doenst work
+    # def action(self, obj):
+    #     # csrfmiddlewaretoken: Wa8UcQ4fD3FJibzxqHN3IYrrjLo4VguWynmbzzcPYoebfVUnDovon7GEMYFRgsh0
+    #     # action: update_snapshots
+    #     # select_across: 0
+    #     # _selected_action: 76d29b26-2a88-439e-877c-a7cca1b72bb3
+    #     return format_html(
+    #         '''
+    #             <form action="/admin/core/snapshot/" method="post" onsubmit="e => e.stopPropagation()">
+    #                 <input type="hidden" name="csrfmiddlewaretoken" value="{}">
+    #                 <input type="hidden" name="_selected_action" value="{}">
+    #                 <button name="update_snapshots">Check</button>
+    #                 <button name="update_titles">Pull title + favicon</button>
+    #                 <button name="update_snapshots">Update</button>
+    #                 <button name="overwrite_snapshots">Re-Archive (overwrite)</button>
+    #                 <button name="delete_snapshots">Permanently delete</button>
+    #             </form>
+    #         ''',
+    #         csrf.get_token(self.request),
+    #         obj.pk,
+    #     )
+
+    def admin_actions(self, obj):
+        return format_html(
+            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+            '''
+            <a class="btn" style="font-size: 18px; display: inline-block; border-radius: 10px; border: 3px solid #eee; padding: 4px 8px" href="/archive/{}">Summary page ➡️</a> &nbsp; &nbsp;
+            <a class="btn" style="font-size: 18px; display: inline-block; border-radius: 10px; border: 3px solid #eee; padding: 4px 8px" href="/archive/{}/index.html#all">Result files 📑</a> &nbsp; &nbsp;
+            <a class="btn" style="font-size: 18px; display: inline-block; border-radius: 10px; border: 3px solid #eee; padding: 4px 8px" href="/admin/core/snapshot/?id__exact={}">Admin actions ⚙️</a>
+            ''',
+            obj.timestamp,
+            obj.timestamp,
+            obj.pk,
+        )
+
+    def status_info(self, obj):
+        return format_html(
+            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
+            '''
+            Archived: {} ({} files {}) &nbsp; &nbsp;
+            Favicon: <img src="{}" style="height: 20px"/> &nbsp; &nbsp;
+            Status code: {} &nbsp; &nbsp;<br/>
+            Server: {} &nbsp; &nbsp;
+            Content type: {} &nbsp; &nbsp;
+            Extension: {} &nbsp; &nbsp;
+            ''',
+            '✅' if obj.is_archived else '❌',
+            obj.num_outputs,
+            self.size(obj) or '0kb',
+            f'/archive/{obj.timestamp}/favicon.ico',
+            obj.status_code or '-',
+            obj.headers and obj.headers.get('Server') or '-',
+            obj.headers and obj.headers.get('Content-Type') or '-',
+            obj.extension or '-',
+        )
+
+    def API(self, obj):
+        try:
+            return get_abid_info(self, obj)
+        except Exception as e:
+            return str(e)
+
+    @admin.display(
+        description='Title',
+        ordering='title',
+    )
+    def title_str(self, obj):
+        tags = ''.join(
+            format_html('<a href="/admin/core/snapshot/?tags__id__exact={}"><span class="tag">{}</span></a> ', tag.pk, tag.name)
+            for tag in obj.tags.all()
+            if str(tag.name).strip()
+        )
+        return format_html(
+            '<a href="/{}">'
+                '<img src="/{}/favicon.ico" class="favicon" onerror="this.remove()">'
+            '</a>'
+            '<a href="/{}/index.html">'
+                '<b class="status-{}">{}</b>'
+            '</a>',
+            obj.archive_path,
+            obj.archive_path,
+            obj.archive_path,
+            'fetched' if obj.latest_title or obj.title else 'pending',
+            urldecode(htmldecode(obj.latest_title or obj.title or ''))[:128] or 'Pending...'
+        ) + mark_safe(f' <span class="tags">{tags}</span>')
+
+    @admin.display(
+        description='Files Saved',
+        # ordering='archiveresult_count',
+    )
+    def files(self, obj):
+        return snapshot_icons(obj)
+
+
+    @admin.display(
+        # ordering='archiveresult_count'
+    )
+    def size(self, obj):
+        archive_size = (Path(obj.link_dir) / 'index.html').exists() and obj.archive_size
+        if archive_size:
+            size_txt = printable_filesize(archive_size)
+            if archive_size > 52428800:
+                size_txt = mark_safe(f'<b>{size_txt}</b>')
+        else:
+            size_txt = mark_safe('<span style="opacity: 0.3">...</span>')
+        return format_html(
+            '<a href="/{}" title="View all files">{}</a>',
+            obj.archive_path,
+            size_txt,
+        )
+
+
+    @admin.display(
+        description='Original URL',
+        ordering='url',
+    )
+    def url_str(self, obj):
+        return format_html(
+            '<a href="{}"><code style="user-select: all;">{}</code></a>',
+            obj.url,
+            obj.url[:128],
+        )
+
+    def grid_view(self, request, extra_context=None):
+
+        # cl = self.get_changelist_instance(request)
+
+        # Save before monkey patching to restore for changelist list view
+        saved_change_list_template = self.change_list_template
+        saved_list_per_page = self.list_per_page
+        saved_list_max_show_all = self.list_max_show_all
+
+        # Monkey patch here plus core_tags.py
+        self.change_list_template = 'private_index_grid.html'
+        self.list_per_page = CONFIG.SNAPSHOTS_PER_PAGE
+        self.list_max_show_all = self.list_per_page
+
+        # Call monkey patched view
+        rendered_response = self.changelist_view(request, extra_context=extra_context)
+
+        # Restore values
+        self.change_list_template = saved_change_list_template
+        self.list_per_page = saved_list_per_page
+        self.list_max_show_all = saved_list_max_show_all
+
+        return rendered_response
+
+    # for debugging, uncomment this to print all requests:
+    # def changelist_view(self, request, extra_context=None):
+    #     print('[*] Got request', request.method, request.POST)
+    #     return super().changelist_view(request, extra_context=None)
+
+    @admin.action(
+        description="Pull"
+    )
+    def update_snapshots(self, request, queryset):
+        archive_links([
+            snapshot.as_link()
+            for snapshot in queryset
+        ], out_dir=CONFIG.OUTPUT_DIR)
+
+    @admin.action(
+        description="⬇️ Title"
+    )
+    def update_titles(self, request, queryset):
+        archive_links([
+            snapshot.as_link()
+            for snapshot in queryset
+        ], overwrite=True, methods=('title','favicon'), out_dir=CONFIG.OUTPUT_DIR)
+
+    @admin.action(
+        description="Re-Snapshot"
+    )
+    def resnapshot_snapshot(self, request, queryset):
+        for snapshot in queryset:
+            timestamp = datetime.now(timezone.utc).isoformat('T', 'seconds')
+            new_url = snapshot.url.split('#')[0] + f'#{timestamp}'
+            add(new_url, tag=snapshot.tags_str())
+
+    @admin.action(
+        description="Reset"
+    )
+    def overwrite_snapshots(self, request, queryset):
+        archive_links([
+            snapshot.as_link()
+            for snapshot in queryset
+        ], overwrite=True, out_dir=CONFIG.OUTPUT_DIR)
+
+    @admin.action(
+        description="Delete"
+    )
+    def delete_snapshots(self, request, queryset):
+        remove(snapshots=queryset, yes=True, delete=True, out_dir=CONFIG.OUTPUT_DIR)
+
+
+    @admin.action(
+        description="+"
+    )
+    def add_tags(self, request, queryset):
+        tags = request.POST.getlist('tags')
+        print('[+] Adding tags', tags, 'to Snapshots', queryset)
+        for obj in queryset:
+            obj.tags.add(*tags)
+
+
+    @admin.action(
+        description="–"
+    )
+    def remove_tags(self, request, queryset):
+        tags = request.POST.getlist('tags')
+        print('[-] Removing tags', tags, 'to Snapshots', queryset)
+        for obj in queryset:
+            obj.tags.remove(*tags)
+
+
+        
+
+
+
+# @admin.register(SnapshotTag, site=archivebox_admin)
+# class SnapshotTagAdmin(ABIDModelAdmin):
+#     list_display = ('id', 'snapshot', 'tag')
+#     sort_fields = ('id', 'snapshot', 'tag')
+#     search_fields = ('id', 'snapshot_id', 'tag_id')
+#     fields = ('snapshot', 'id')
+#     actions = ['delete_selected']
+#     ordering = ['-id']
+
+#     def API(self, obj):
+#         return get_abid_info(self, obj)
+
+
+@admin.register(Tag, site=archivebox_admin)
+class TagAdmin(ABIDModelAdmin):
+    list_display = ('created', 'created_by', 'abid', 'name', 'num_snapshots', 'snapshots')
+    sort_fields = ('name', 'slug', 'abid', 'created_by', 'created')
+    readonly_fields = ('slug', 'abid', 'created', 'modified', 'API', 'num_snapshots', 'snapshots')
+    search_fields = ('abid', 'name', 'slug')
+    fields = ('name', 'created_by', *readonly_fields)
+    actions = ['delete_selected']
+    ordering = ['-created']
+
+    paginator = AccelleratedPaginator
+
+    def API(self, obj):
+        try:
+            return get_abid_info(self, obj)
+        except Exception as e:
+            return str(e)
+
+    def num_snapshots(self, tag):
+        return format_html(
+            '<a href="/admin/core/snapshot/?tags__id__exact={}">{} total</a>',
+            tag.id,
+            tag.snapshot_set.count(),
+        )
+
+    def snapshots(self, tag):
+        total_count = tag.snapshot_set.count()
+        return mark_safe('<br/>'.join(
+            format_html(
+                '<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> {}',
+                snap.pk,
+                snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
+                snap.url[:64],
+            )
+            for snap in tag.snapshot_set.order_by('-updated')[:10]
+        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={tag.id}">and {total_count-10} more...<a>' if tag.snapshot_set.count() > 10 else ''))
+
+
+@admin.register(ArchiveResult, site=archivebox_admin)
+class ArchiveResultAdmin(ABIDModelAdmin):
+    list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
+    sort_fields = ('start_ts', 'extractor', 'status')
+    readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created', 'modified', 'API', 'output_summary')
+    search_fields = ('id', 'old_id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
+    fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'start_ts', 'end_ts', 'created_by', 'cmd_version', 'cmd', *readonly_fields)
+    autocomplete_fields = ['snapshot']
+
+    list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
+    ordering = ['-start_ts']
+    list_per_page = CONFIG.SNAPSHOTS_PER_PAGE
+    
+    paginator = AccelleratedPaginator
+
+    @admin.display(
+        description='Snapshot Info'
+    )
+    def snapshot_info(self, result):
+        return format_html(
+            '<a href="/archive/{}/index.html"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
+            result.snapshot.timestamp,
+            result.snapshot.abid,
+            result.snapshot.added.strftime('%Y-%m-%d %H:%M'),
+            result.snapshot.url[:128],
+        )
+
+    def API(self, obj):
+        try:
+            return get_abid_info(self, obj)
+        except Exception as e:
+            raise e
+            return str(e)
+
+    @admin.display(
+        description='Snapshot Tags'
+    )
+    def tags_str(self, result):
+        return result.snapshot.tags_str()
+
+    def cmd_str(self, result):
+        return format_html(
+            '<pre>{}</pre>',
+            ' '.join(result.cmd) if isinstance(result.cmd, list) else str(result.cmd),
+        )
+    
+    def output_str(self, result):
+        return format_html(
+            '<a href="/archive/{}/{}" class="output-link">↗️</a><pre>{}</pre>',
+            result.snapshot.timestamp,
+            result.output if (result.status == 'succeeded') and result.extractor not in ('title', 'archive_org') else 'index.html',
+            result.output,
+        )
+
+    def output_summary(self, result):
+        snapshot_dir = Path(CONFIG.OUTPUT_DIR) / str(result.pwd).split('data/', 1)[-1]
+        output_str = format_html(
+            '<pre style="display: block; max-width: 75vw; white-space: pre-line;">{}</pre><br/>',
+            result.output,
+        )
+        output_str += format_html('<a href="/archive/{}/index.html#all">See result files ...</a><br/><pre><code>', str(result.snapshot.timestamp))
+        path_from_output_str = (snapshot_dir / result.output)
+        output_str += format_html('<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>', str(snapshot_dir), str(result.output))
+        
+        path_exists = False
+        try:
+            path_exists = path_from_output_str.exists()
+        except OSError:
+            pass
+
+        if path_exists:
+            root_dir = str(path_from_output_str)
+        else:
+            root_dir = str(snapshot_dir)
+
+        # print(root_dir, str(list(os.walk(root_dir))))
+
+        for root, dirs, files in os.walk(root_dir):
+            depth = root.replace(root_dir, '').count(os.sep) + 1
+            if depth > 2:
+                continue
+            indent = ' ' * 4 * (depth)
+            output_str += format_html('<b style="padding: 1px">{}{}/</b><br/>', indent, os.path.basename(root))
+            indentation_str = ' ' * 4 * (depth + 1)
+            for filename in sorted(files):
+                is_hidden = filename.startswith('.')
+                output_str += format_html('<span style="opacity: {}.2">{}{}</span><br/>', int(not is_hidden), indentation_str, filename.strip())
+
+        return output_str + format_html('</code></pre>')
+
+
+
+@admin.register(APIToken, site=archivebox_admin)
+class APITokenAdmin(ABIDModelAdmin):
+    list_display = ('created', 'abid', 'created_by', 'token_redacted', 'expires')
+    sort_fields = ('abid', 'created', 'created_by', 'expires')
+    readonly_fields = ('abid', 'created')
+    search_fields = ('id', 'abid', 'created_by__username', 'token')
+    fields = ('created_by', 'token', 'expires', *readonly_fields)
+
+    list_filter = ('created_by',)
+    ordering = ['-created']
+    list_per_page = 100
+
+@admin.register(get_webhook_model(), site=archivebox_admin)
+class CustomWebhookAdmin(WebhookAdmin, ABIDModelAdmin):
+    list_display = ('created', 'created_by', 'abid', *WebhookAdmin.list_display)
+    sort_fields = ('created', 'created_by', 'abid', 'referenced_model', 'endpoint', 'last_success', 'last_error')
+    readonly_fields = ('abid', 'created', *WebhookAdmin.readonly_fields)
@@ -1,199 +0,0 @@
-__package__ = 'archivebox.core'
-
-import os
-from pathlib import Path
-
-from django.contrib import admin
-from django.utils.html import format_html, mark_safe
-from django.core.exceptions import ValidationError
-from django.urls import reverse, resolve
-from django.utils import timezone
-from django_jsonform.forms.fields import JSONFormField
-
-from huey_monitor.admin import TaskModel
-
-import abx
-
-from archivebox.config import DATA_DIR
-from archivebox.config.common import SERVER_CONFIG
-from archivebox.misc.paginators import AccelleratedPaginator
-
-from abid_utils.admin import ABIDModelAdmin
-
-from core.models import ArchiveResult, Snapshot
-
-
-
-
-def result_url(result: TaskModel) -> str:
-    url = reverse("admin:huey_monitor_taskmodel_change", args=[str(result.id)])
-    return format_html('<a href="{url}" class="fade-in-progress-url">See progress...</a>'.format(url=url))
-
-
-
-class ArchiveResultInline(admin.TabularInline):
-    name = 'Archive Results Log'
-    model = ArchiveResult
-    parent_model = Snapshot
-    # fk_name = 'snapshot'
-    extra = 0
-    sort_fields = ('end_ts', 'extractor', 'output', 'status', 'cmd_version')
-    readonly_fields = ('id', 'result_id', 'completed', 'command', 'version')
-    fields = ('start_ts', 'end_ts', *readonly_fields, 'extractor', 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
-    # exclude = ('id',)
-    ordering = ('end_ts',)
-    show_change_link = True
-    # # classes = ['collapse']
-    # # list_display_links = ['abid']
-
-    def get_parent_object_from_request(self, request):
-        resolved = resolve(request.path_info)
-        try:
-            return self.parent_model.objects.get(pk=resolved.kwargs['object_id'])
-        except (self.parent_model.DoesNotExist, ValidationError):
-            return self.parent_model.objects.get(pk=self.parent_model.id_from_abid(resolved.kwargs['object_id']))
-
-    @admin.display(
-        description='Completed',
-        ordering='end_ts',
-    )
-    def completed(self, obj):
-        return format_html('<p style="white-space: nowrap">{}</p>', obj.end_ts.strftime('%Y-%m-%d %H:%M:%S'))
-
-    def result_id(self, obj):
-        return format_html('<a href="{}"><code style="font-size: 10px">[{}]</code></a>', reverse('admin:core_archiveresult_change', args=(obj.id,)), obj.abid)
-    
-    def command(self, obj):
-        return format_html('<small><code>{}</code></small>', " ".join(obj.cmd or []))
-    
-    def version(self, obj):
-        return format_html('<small><code>{}</code></small>', obj.cmd_version or '-')
-    
-    def get_formset(self, request, obj=None, **kwargs):
-        formset = super().get_formset(request, obj, **kwargs)
-        snapshot = self.get_parent_object_from_request(request)
-
-        # import ipdb; ipdb.set_trace()
-        # formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
-        
-        # default values for new entries
-        formset.form.base_fields['status'].initial = 'succeeded'
-        formset.form.base_fields['start_ts'].initial = timezone.now()
-        formset.form.base_fields['end_ts'].initial = timezone.now()
-        formset.form.base_fields['cmd_version'].initial = '-'
-        formset.form.base_fields['pwd'].initial = str(snapshot.link_dir)
-        formset.form.base_fields['created_by'].initial = request.user
-        formset.form.base_fields['cmd'] = JSONFormField(initial=['-'])
-        formset.form.base_fields['output'].initial = 'Manually recorded cmd output...'
-        
-        if obj is not None:
-            # hidden values for existing entries and new entries
-            formset.form.base_fields['start_ts'].widget = formset.form.base_fields['start_ts'].hidden_widget()
-            formset.form.base_fields['end_ts'].widget = formset.form.base_fields['end_ts'].hidden_widget()
-            formset.form.base_fields['cmd'].widget = formset.form.base_fields['cmd'].hidden_widget()
-            formset.form.base_fields['pwd'].widget = formset.form.base_fields['pwd'].hidden_widget()
-            formset.form.base_fields['created_by'].widget = formset.form.base_fields['created_by'].hidden_widget()
-            formset.form.base_fields['cmd_version'].widget = formset.form.base_fields['cmd_version'].hidden_widget()
-        return formset
-    
-    def get_readonly_fields(self, request, obj=None):
-        if obj is not None:
-            return self.readonly_fields
-        else:
-            return []
-
-
-
-class ArchiveResultAdmin(ABIDModelAdmin):
-    list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
-    sort_fields = ('start_ts', 'extractor', 'status')
-    readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created_at', 'modified_at', 'abid_info', 'output_summary')
-    search_fields = ('id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
-    fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'start_ts', 'end_ts', 'created_by', 'cmd_version', 'cmd', *readonly_fields)
-    autocomplete_fields = ['snapshot']
-
-    list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
-    ordering = ['-start_ts']
-    list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE
-    
-    paginator = AccelleratedPaginator
-    save_on_top = True
-    
-    actions = ['delete_selected']
-    
-    class Meta:
-        verbose_name = 'Archive Result'
-        verbose_name_plural = 'Archive Results'
-
-    def change_view(self, request, object_id, form_url="", extra_context=None):
-        self.request = request
-        return super().change_view(request, object_id, form_url, extra_context)
-
-    @admin.display(
-        description='Snapshot Info'
-    )
-    def snapshot_info(self, result):
-        return format_html(
-            '<a href="/archive/{}/index.html"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            result.snapshot.timestamp,
-            result.snapshot.abid,
-            result.snapshot.bookmarked_at.strftime('%Y-%m-%d %H:%M'),
-            result.snapshot.url[:128],
-        )
-
-
-    @admin.display(
-        description='Snapshot Tags'
-    )
-    def tags_str(self, result):
-        return result.snapshot.tags_str()
-
-    def cmd_str(self, result):
-        return format_html(
-            '<pre>{}</pre>',
-            ' '.join(result.cmd) if isinstance(result.cmd, list) else str(result.cmd),
-        )
-    
-    def output_str(self, result):
-        return format_html(
-            '<a href="/archive/{}/{}" class="output-link">↗️</a><pre>{}</pre>',
-            result.snapshot.timestamp,
-            result.output if (result.status == 'succeeded') and result.extractor not in ('title', 'archive_org') else 'index.html',
-            result.output,
-        )
-
-    def output_summary(self, result):
-        snapshot_dir = Path(DATA_DIR) / str(result.pwd).split('data/', 1)[-1]
-        output_str = format_html(
-            '<pre style="display: inline-block">{}</pre><br/>',
-            result.output,
-        )
-        output_str += format_html('<a href="/archive/{}/index.html#all">See result files ...</a><br/><pre><code>', str(result.snapshot.timestamp))
-        path_from_output_str = (snapshot_dir / result.output)
-        output_str += format_html('<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>', str(snapshot_dir), str(result.output))
-        if os.access(path_from_output_str, os.R_OK):
-            root_dir = str(path_from_output_str)
-        else:
-            root_dir = str(snapshot_dir)
-
-        # print(root_dir, str(list(os.walk(root_dir))))
-
-        for root, dirs, files in os.walk(root_dir):
-            depth = root.replace(root_dir, '').count(os.sep) + 1
-            if depth > 2:
-                continue
-            indent = ' ' * 4 * (depth)
-            output_str += format_html('<b style="padding: 1px">{}{}/</b><br/>', indent, os.path.basename(root))
-            indentation_str = ' ' * 4 * (depth + 1)
-            for filename in sorted(files):
-                is_hidden = filename.startswith('.')
-                output_str += format_html('<span style="opacity: {}.2">{}{}</span><br/>', int(not is_hidden), indentation_str, filename.strip())
-
-        return output_str + format_html('</code></pre>')
-
-
-
-
-@abx.hookimpl
-def register_admin(admin_site):
-    admin_site.register(ArchiveResult, ArchiveResultAdmin)
@@ -1,42 +0,0 @@
-__package__ = 'archivebox.core'
-
-from django.contrib import admin
-
-import archivebox
-
-class ArchiveBoxAdmin(admin.AdminSite):
-    site_header = 'ArchiveBox'
-    index_title = 'Admin Views'
-    site_title = 'Admin'
-    namespace = 'admin'
-
-
-archivebox_admin = ArchiveBoxAdmin()
-archivebox_admin.disable_action('delete_selected')
-# TODO: https://stackoverflow.com/questions/40760880/add-custom-button-to-django-admin-panel
-
-
-
-# patch admin with methods to add data views (implemented by admin_data_views package)
-# https://github.com/MrThearMan/django-admin-data-views
-# https://mrthearman.github.io/django-admin-data-views/setup/
-from admin_data_views.admin import get_app_list, admin_data_index_view, get_admin_data_urls, get_urls
-archivebox_admin.get_app_list = get_app_list.__get__(archivebox_admin, ArchiveBoxAdmin)
-archivebox_admin.admin_data_index_view = admin_data_index_view.__get__(archivebox_admin, ArchiveBoxAdmin)       # type: ignore
-archivebox_admin.get_admin_data_urls = get_admin_data_urls.__get__(archivebox_admin, ArchiveBoxAdmin)           # type: ignore
-archivebox_admin.get_urls = get_urls(archivebox_admin.get_urls).__get__(archivebox_admin, ArchiveBoxAdmin)
-############### Admin Data View sections are defined in settings.ADMIN_DATA_VIEWS #########
-
-
-def register_admin_site():
-    """Replace the default admin site with our custom ArchiveBox admin site."""
-    from django.contrib import admin
-    from django.contrib.admin import sites
-
-    admin.site = archivebox_admin
-    sites.site = archivebox_admin
-    
-    # register all plugins admin classes
-    archivebox.pm.hook.register_admin(admin_site=archivebox_admin)
-    
-    return archivebox_admin
@@ -1,357 +0,0 @@
-
-__package__ = 'archivebox.core'
-
-import os
-from pathlib import Path
-
-from django.contrib import admin, messages
-from django.urls import path
-from django.utils.html import format_html, mark_safe
-from django.utils import timezone
-from django import forms
-from django.template import Template, RequestContext
-from django.contrib.admin.helpers import ActionForm
-from django.contrib.admin.widgets import FilteredSelectMultiple
-
-from archivebox.config import DATA_DIR
-from archivebox.config.common import SERVER_CONFIG
-from archivebox.misc.util import htmldecode, urldecode
-from archivebox.misc.paginators import AccelleratedPaginator
-from archivebox.search.admin import SearchResultsAdminMixin
-
-from archivebox.logging_util import printable_filesize
-from archivebox.index.html import snapshot_icons
-from archivebox.extractors import archive_links
-from archivebox.main import remove
-
-from archivebox.abid_utils.admin import ABIDModelAdmin
-from archivebox.queues.tasks import bg_archive_links, bg_add
-
-from core.models import Tag
-from core.admin_tags import TagInline
-from core.admin_archiveresults import ArchiveResultInline, result_url
-
-
-# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
-GLOBAL_CONTEXT = {}
-
-
-class SnapshotActionForm(ActionForm):
-    tags = forms.ModelMultipleChoiceField(
-        label='Edit tags',
-        queryset=Tag.objects.all(),
-        required=False,
-        widget=FilteredSelectMultiple(
-            'core_tag__name',
-            False,
-        ),
-    )
-
-    # TODO: allow selecting actions for specific extractors? is this useful?
-    # extractor = forms.ChoiceField(
-    #     choices=ArchiveResult.EXTRACTOR_CHOICES,
-    #     required=False,
-    #     widget=forms.MultileChoiceField(attrs={'class': "form-control"})
-    # )
-
-
-class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
-    list_display = ('created_at', 'title_str', 'files', 'size', 'url_str')
-    sort_fields = ('title_str', 'url_str', 'created_at')
-    readonly_fields = ('admin_actions', 'status_info', 'tags_str', 'imported_timestamp', 'created_at', 'modified_at', 'downloaded_at', 'abid_info', 'link_dir')
-    search_fields = ('id', 'url', 'abid', 'timestamp', 'title', 'tags__name')
-    list_filter = ('created_at', 'downloaded_at', 'archiveresult__status', 'created_by', 'tags__name')
-    fields = ('url', 'title', 'created_by', 'bookmarked_at', *readonly_fields)
-    ordering = ['-created_at']
-    actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
-    inlines = [TagInline, ArchiveResultInline]
-    list_per_page = min(max(5, SERVER_CONFIG.SNAPSHOTS_PER_PAGE), 5000)
-
-    action_form = SnapshotActionForm
-    paginator = AccelleratedPaginator
-
-    save_on_top = True
-    show_full_result_count = False
-
-    def changelist_view(self, request, extra_context=None):
-        self.request = request
-        extra_context = extra_context or {}
-        try:
-            return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)
-        except Exception as e:
-            self.message_user(request, f'Error occurred while loading the page: {str(e)} {request.GET} {request.POST}')
-            return super().changelist_view(request, GLOBAL_CONTEXT)
-
-
-    def get_urls(self):
-        urls = super().get_urls()
-        custom_urls = [
-            path('grid/', self.admin_site.admin_view(self.grid_view), name='grid')
-        ]
-        return custom_urls + urls
-
-    # def get_queryset(self, request):
-    #     # tags_qs = SnapshotTag.objects.all().select_related('tag')
-    #     # prefetch = Prefetch('snapshottag_set', queryset=tags_qs)
-
-    #     self.request = request
-    #     return super().get_queryset(request).prefetch_related('archiveresult_set').distinct()  # .annotate(archiveresult_count=Count('archiveresult'))
-
-    @admin.action(
-        description="Imported Timestamp"
-    )
-    def imported_timestamp(self, obj):
-        context = RequestContext(self.request, {
-            'bookmarked_date': obj.bookmarked,
-            'timestamp': obj.timestamp,
-        })
-
-        html = Template("""{{bookmarked_date}} (<code>{{timestamp}}</code>)""")
-        return mark_safe(html.render(context))
-    
-        # pretty_time = obj.bookmarked.strftime('%Y-%m-%d %H:%M:%S')
-        # return f'{pretty_time} ({obj.timestamp})'
-
-    # TODO: figure out a different way to do this, you cant nest forms so this doenst work
-    # def action(self, obj):
-    #     # csrfmiddlewaretoken: Wa8UcQ4fD3FJibzxqHN3IYrrjLo4VguWynmbzzcPYoebfVUnDovon7GEMYFRgsh0
-    #     # action: update_snapshots
-    #     # select_across: 0
-    #     # _selected_action: 76d29b26-2a88-439e-877c-a7cca1b72bb3
-    #     return format_html(
-    #         '''
-    #             <form action="/admin/core/snapshot/" method="post" onsubmit="e => e.stopPropagation()">
-    #                 <input type="hidden" name="csrfmiddlewaretoken" value="{}">
-    #                 <input type="hidden" name="_selected_action" value="{}">
-    #                 <button name="update_snapshots">Check</button>
-    #                 <button name="update_titles">Pull title + favicon</button>
-    #                 <button name="update_snapshots">Update</button>
-    #                 <button name="overwrite_snapshots">Re-Archive (overwrite)</button>
-    #                 <button name="delete_snapshots">Permanently delete</button>
-    #             </form>
-    #         ''',
-    #         csrf.get_token(self.request),
-    #         obj.pk,
-    #     )
-
-    def admin_actions(self, obj):
-        return format_html(
-            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
-            '''
-            <a class="btn" style="font-size: 18px; display: inline-block; border-radius: 10px; border: 3px solid #eee; padding: 4px 8px" href="/archive/{}">Summary page ➡️</a> &nbsp; &nbsp;
-            <a class="btn" style="font-size: 18px; display: inline-block; border-radius: 10px; border: 3px solid #eee; padding: 4px 8px" href="/archive/{}/index.html#all">Result files 📑</a> &nbsp; &nbsp;
-            <a class="btn" style="font-size: 18px; display: inline-block; border-radius: 10px; border: 3px solid #eee; padding: 4px 8px" href="/admin/core/snapshot/?id__exact={}">Admin actions ⚙️</a>
-            ''',
-            obj.timestamp,
-            obj.timestamp,
-            obj.pk,
-        )
-
-    def status_info(self, obj):
-        return format_html(
-            # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
-            '''
-            Archived: {} ({} files {}) &nbsp; &nbsp;
-            Favicon: <img src="{}" style="height: 20px"/> &nbsp; &nbsp;
-            Status code: {} &nbsp; &nbsp;<br/>
-            Server: {} &nbsp; &nbsp;
-            Content type: {} &nbsp; &nbsp;
-            Extension: {} &nbsp; &nbsp;
-            ''',
-            '✅' if obj.is_archived else '❌',
-            obj.num_outputs,
-            self.size(obj) or '0kb',
-            f'/archive/{obj.timestamp}/favicon.ico',
-            obj.status_code or '-',
-            obj.headers and obj.headers.get('Server') or '-',
-            obj.headers and obj.headers.get('Content-Type') or '-',
-            obj.extension or '-',
-        )
-
-    @admin.display(
-        description='Title',
-        ordering='title',
-    )
-    def title_str(self, obj):
-        tags = ''.join(
-            format_html('<a href="/admin/core/snapshot/?tags__id__exact={}"><span class="tag">{}</span></a> ', tag.pk, tag.name)
-            for tag in obj.tags.all()
-            if str(tag.name).strip()
-        )
-        return format_html(
-            '<a href="/{}">'
-                '<img src="/{}/favicon.ico" class="favicon" onerror="this.remove()">'
-            '</a>'
-            '<a href="/{}/index.html">'
-                '<b class="status-{}">{}</b>'
-            '</a>',
-            obj.archive_path,
-            obj.archive_path,
-            obj.archive_path,
-            'fetched' if obj.latest_title or obj.title else 'pending',
-            urldecode(htmldecode(obj.latest_title or obj.title or ''))[:128] or 'Pending...'
-        ) + mark_safe(f' <span class="tags">{tags}</span>')
-
-    @admin.display(
-        description='Files Saved',
-        # ordering='archiveresult_count',
-    )
-    def files(self, obj):
-        # return '-'
-        return snapshot_icons(obj)
-
-
-    @admin.display(
-        # ordering='archiveresult_count'
-    )
-    def size(self, obj):
-        archive_size = os.access(Path(obj.link_dir) / 'index.html', os.F_OK) and obj.archive_size
-        if archive_size:
-            size_txt = printable_filesize(archive_size)
-            if archive_size > 52428800:
-                size_txt = mark_safe(f'<b>{size_txt}</b>')
-        else:
-            size_txt = mark_safe('<span style="opacity: 0.3">...</span>')
-        return format_html(
-            '<a href="/{}" title="View all files">{}</a>',
-            obj.archive_path,
-            size_txt,
-        )
-
-
-    @admin.display(
-        description='Original URL',
-        ordering='url',
-    )
-    def url_str(self, obj):
-        return format_html(
-            '<a href="{}"><code style="user-select: all;">{}</code></a>',
-            obj.url,
-            obj.url[:128],
-        )
-
-    def grid_view(self, request, extra_context=None):
-
-        # cl = self.get_changelist_instance(request)
-
-        # Save before monkey patching to restore for changelist list view
-        saved_change_list_template = self.change_list_template
-        saved_list_per_page = self.list_per_page
-        saved_list_max_show_all = self.list_max_show_all
-
-        # Monkey patch here plus core_tags.py
-        self.change_list_template = 'private_index_grid.html'
-        self.list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE
-        self.list_max_show_all = self.list_per_page
-
-        # Call monkey patched view
-        rendered_response = self.changelist_view(request, extra_context=extra_context)
-
-        # Restore values
-        self.change_list_template = saved_change_list_template
-        self.list_per_page = saved_list_per_page
-        self.list_max_show_all = saved_list_max_show_all
-
-        return rendered_response
-
-    # for debugging, uncomment this to print all requests:
-    # def changelist_view(self, request, extra_context=None):
-    #     print('[*] Got request', request.method, request.POST)
-    #     return super().changelist_view(request, extra_context=None)
-
-    @admin.action(
-        description="ℹ️ Get Title"
-    )
-    def update_titles(self, request, queryset):
-        links = [snapshot.as_link() for snapshot in queryset]
-        if len(links) < 3:
-            # run syncronously if there are only 1 or 2 links
-            archive_links(links, overwrite=True, methods=('title','favicon'), out_dir=DATA_DIR)
-            messages.success(request, f"Title and favicon have been fetched and saved for {len(links)} URLs.")
-        else:
-            # otherwise run in a background worker
-            result = bg_archive_links((links,), kwargs={"overwrite": True, "methods": ["title", "favicon"], "out_dir": DATA_DIR})
-            messages.success(
-                request,
-                mark_safe(f"Title and favicon are updating in the background for {len(links)} URLs. {result_url(result)}"),
-            )
-
-    @admin.action(
-        description="⬇️ Get Missing"
-    )
-    def update_snapshots(self, request, queryset):
-        links = [snapshot.as_link() for snapshot in queryset]
-
-        result = bg_archive_links((links,), kwargs={"overwrite": False, "out_dir": DATA_DIR})
-
-        messages.success(
-            request,
-            mark_safe(f"Re-trying any previously failed methods for {len(links)} URLs in the background. {result_url(result)}"),
-        )
-
-
-    @admin.action(
-        description="🆕 Archive Again"
-    )
-    def resnapshot_snapshot(self, request, queryset):
-        for snapshot in queryset:
-            timestamp = timezone.now().isoformat('T', 'seconds')
-            new_url = snapshot.url.split('#')[0] + f'#{timestamp}'
-
-            result = bg_add({'urls': new_url, 'tag': snapshot.tags_str()})
-
-        messages.success(
-            request,
-            mark_safe(f"Creating new fresh snapshots for {queryset.count()} URLs in the background. {result_url(result)}"),
-        )
-
-    @admin.action(
-        description="🔄 Redo"
-    )
-    def overwrite_snapshots(self, request, queryset):
-        links = [snapshot.as_link() for snapshot in queryset]
-
-        result = bg_archive_links((links,), kwargs={"overwrite": True, "out_dir": DATA_DIR})
-
-        messages.success(
-            request,
-            mark_safe(f"Clearing all previous results and re-downloading {len(links)} URLs in the background. {result_url(result)}"),
-        )
-
-    @admin.action(
-        description="☠️ Delete"
-    )
-    def delete_snapshots(self, request, queryset):
-        remove(snapshots=queryset, yes=True, delete=True, out_dir=DATA_DIR)
-        messages.success(
-            request,
-            mark_safe(f"Succesfully deleted {queryset.count()} Snapshots. Don't forget to scrub URLs from import logs (data/sources) and error logs (data/logs) if needed."),
-        )
-
-
-    @admin.action(
-        description="+"
-    )
-    def add_tags(self, request, queryset):
-        tags = request.POST.getlist('tags')
-        print('[+] Adding tags', tags, 'to Snapshots', queryset)
-        for obj in queryset:
-            obj.tags.add(*tags)
-        messages.success(
-            request,
-            f"Added {len(tags)} tags to {queryset.count()} Snapshots.",
-        )
-
-
-    @admin.action(
-        description="–"
-    )
-    def remove_tags(self, request, queryset):
-        tags = request.POST.getlist('tags')
-        print('[-] Removing tags', tags, 'to Snapshots', queryset)
-        for obj in queryset:
-            obj.tags.remove(*tags)
-        messages.success(
-            request,
-            f"Removed {len(tags)} tags from {queryset.count()} Snapshots.",
-        )
@@ -1,165 +0,0 @@
-__package__ = 'archivebox.core'
-
-from django.contrib import admin
-from django.utils.html import format_html, mark_safe
-
-import abx
-
-from abid_utils.admin import ABIDModelAdmin
-from archivebox.misc.paginators import AccelleratedPaginator
-
-from core.models import Tag
-
-
-class TagInline(admin.TabularInline):
-    model = Tag.snapshot_set.through       # type: ignore
-    # fk_name = 'snapshot'
-    fields = ('id', 'tag')
-    extra = 1
-    # min_num = 1
-    max_num = 1000
-    autocomplete_fields = (
-        'tag',
-    )
-    
-
-# class AutocompleteTags:
-#     model = Tag
-#     search_fields = ['name']
-#     name = 'name'
-#     # source_field = 'name'
-#     remote_field = Tag._meta.get_field('name')
-
-# class AutocompleteTagsAdminStub:
-#     name = 'admin'
-    
-    
-# class TaggedItemInline(admin.TabularInline):
-#     readonly_fields = ('object_link',)
-#     fields = ('id', 'tag', 'content_type', 'object_id', *readonly_fields)
-#     model = TaggedItem
-#     extra = 1
-#     show_change_link = True
-    
-#     @admin.display(description='object')
-#     def object_link(self, obj):
-#         obj = obj.content_type.get_object_for_this_type(pk=obj.object_id)
-#         return format_html('<a href="/admin/{}/{}/{}/change"><b>[{}]</b></a>', obj._meta.app_label, obj._meta.model_name, obj.pk, str(obj))
-
-    
-class TagAdmin(ABIDModelAdmin):
-    list_display = ('created_at', 'created_by', 'abid', 'name', 'num_snapshots', 'snapshots')
-    list_filter = ('created_at', 'created_by')
-    sort_fields = ('name', 'slug', 'abid', 'created_by', 'created_at')
-    readonly_fields = ('slug', 'abid', 'created_at', 'modified_at', 'abid_info', 'snapshots')
-    search_fields = ('abid', 'name', 'slug')
-    fields = ('name', 'created_by', *readonly_fields)
-    actions = ['delete_selected', 'merge_tags']
-    ordering = ['-created_at']
-    # inlines = [TaggedItemInline]
-
-    paginator = AccelleratedPaginator
-
-
-    def num_snapshots(self, tag):
-        return format_html(
-            '<a href="/admin/core/snapshot/?tags__id__exact={}">{} total</a>',
-            tag.id,
-            tag.snapshot_set.count(),
-        )
-
-    def snapshots(self, tag):
-        total_count = tag.snapshot_set.count()
-        return mark_safe('<br/>'.join(
-            format_html(
-                '<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> {}',
-                snap.pk,
-                snap.downloaded_at.strftime('%Y-%m-%d %H:%M') if snap.downloaded_at else 'pending...',
-                snap.url[:64],
-            )
-            for snap in tag.snapshot_set.order_by('-downloaded_at')[:10]
-        ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={tag.id}">{total_count} total snapshots...<a>'))
-
-    # def get_urls(self):
-    #     urls = super().get_urls()
-    #     custom_urls = [
-    #         path(
-    #             "merge-tags/",
-    #             self.admin_site.admin_view(self.merge_tags_view),
-    #             name="taggit_tag_merge_tags",
-    #         ),
-    #     ]
-    #     return custom_urls + urls
-
-    # @admin.action(description="Merge selected tags")
-    # def merge_tags(self, request, queryset):
-    #     selected = request.POST.getlist(admin.helpers.ACTION_CHECKBOX_NAME)
-    #     if not selected:
-    #         self.message_user(request, "Please select at least one tag.")
-    #         return redirect(request.get_full_path())
-
-    #     selected_tag_ids = ",".join(selected)
-    #     redirect_url = f"{request.get_full_path()}merge-tags/"
-
-    #     request.session["selected_tag_ids"] = selected_tag_ids
-
-    #     return redirect(redirect_url)
-
-    # def merge_tags_view(self, request):
-    #     selected_tag_ids = request.session.get("selected_tag_ids", "").split(",")
-    #     if request.method == "POST":
-    #         form = MergeTagsForm(request.POST)
-    #         if form.is_valid():
-    #             new_tag_name = form.cleaned_data["new_tag_name"]
-    #             new_tag, created = Tag.objects.get_or_create(name=new_tag_name)
-    #             with transaction.atomic():
-    #                 for tag_id in selected_tag_ids:
-    #                     tag = Tag.objects.get(id=tag_id)
-    #                     tagged_items = TaggedItem.objects.filter(tag=tag)
-    #                     for tagged_item in tagged_items:
-    #                         if TaggedItem.objects.filter(
-    #                             tag=new_tag,
-    #                             content_type=tagged_item.content_type,
-    #                             object_id=tagged_item.object_id,
-    #                         ).exists():
-    #                             # we have the new tag as well, so we can just
-    #                             # remove the tag association
-    #                             tagged_item.delete()
-    #                         else:
-    #                             # point this taggedItem to the new one
-    #                             tagged_item.tag = new_tag
-    #                             tagged_item.save()
-                        
-    #                     # delete the old tag
-    #                     if tag.id != new_tag.id:
-    #                         tag.delete()
-
-    #             self.message_user(request, "Tags have been merged", level="success")
-    #             # clear the selected_tag_ids from session after merge is complete
-    #             request.session.pop("selected_tag_ids", None)
-
-    #             return redirect("..")
-    #         else:
-    #             self.message_user(request, "Form is invalid.", level="error")
-
-    #     context = {
-    #         "form": MergeTagsForm(),
-    #         "selected_tag_ids": selected_tag_ids,
-    #     }
-    #     return render(request, "admin/taggit/merge_tags_form.html", context)
-
-
-# @admin.register(SnapshotTag, site=archivebox_admin)
-# class SnapshotTagAdmin(ABIDModelAdmin):
-#     list_display = ('id', 'snapshot', 'tag')
-#     sort_fields = ('id', 'snapshot', 'tag')
-#     search_fields = ('id', 'snapshot_id', 'tag_id')
-#     fields = ('snapshot', 'id')
-#     actions = ['delete_selected']
-#     ordering = ['-id']
-
-
-@abx.hookimpl
-def register_admin(admin_site):
-    admin_site.register(Tag, TagAdmin)
-
@@ -1,91 +0,0 @@
-__package__ = 'archivebox.core'
-
-from django.contrib import admin
-from django.contrib.auth.admin import UserAdmin
-from django.utils.html import format_html, mark_safe
-from django.contrib.auth import get_user_model
-
-import abx
-
-
-class CustomUserAdmin(UserAdmin):
-    sort_fields = ['id', 'email', 'username', 'is_superuser', 'last_login', 'date_joined']
-    list_display = ['username', 'id', 'email', 'is_superuser', 'last_login', 'date_joined']
-    readonly_fields = ('snapshot_set', 'archiveresult_set', 'tag_set', 'apitoken_set', 'outboundwebhook_set')
-    fieldsets = [*UserAdmin.fieldsets, ('Data', {'fields': readonly_fields})]
-
-    @admin.display(description='Snapshots')
-    def snapshot_set(self, obj):
-        total_count = obj.snapshot_set.count()
-        return mark_safe('<br/>'.join(
-            format_html(
-                '<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> <b>📅 {}</b> {}',
-                snap.pk,
-                snap.abid,
-                snap.downloaded_at.strftime('%Y-%m-%d %H:%M') if snap.downloaded_at else 'pending...',
-                snap.url[:64],
-            )
-            for snap in obj.snapshot_set.order_by('-modified_at')[:10]
-        ) + f'<br/><a href="/admin/core/snapshot/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
-
-    @admin.display(description='Archive Result Logs')
-    def archiveresult_set(self, obj):
-        total_count = obj.archiveresult_set.count()
-        return mark_safe('<br/>'.join(
-            format_html(
-                '<code><a href="/admin/core/archiveresult/{}/change"><b>[{}]</b></a></code> <b>📅 {}</b> <b>📄 {}</b> {}',
-                result.pk,
-                result.abid,
-                result.snapshot.downloaded_at.strftime('%Y-%m-%d %H:%M') if result.snapshot.downloaded_at else 'pending...',
-                result.extractor,
-                result.snapshot.url[:64],
-            )
-            for result in obj.archiveresult_set.order_by('-modified_at')[:10]
-        ) + f'<br/><a href="/admin/core/archiveresult/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
-
-    @admin.display(description='Tags')
-    def tag_set(self, obj):
-        total_count = obj.tag_set.count()
-        return mark_safe(', '.join(
-            format_html(
-                '<code><a href="/admin/core/tag/{}/change"><b>{}</b></a></code>',
-                tag.pk,
-                tag.name,
-            )
-            for tag in obj.tag_set.order_by('-modified_at')[:10]
-        ) + f'<br/><a href="/admin/core/tag/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
-
-    @admin.display(description='API Tokens')
-    def apitoken_set(self, obj):
-        total_count = obj.apitoken_set.count()
-        return mark_safe('<br/>'.join(
-            format_html(
-                '<code><a href="/admin/api/apitoken/{}/change"><b>[{}]</b></a></code> {} (expires {})',
-                apitoken.pk,
-                apitoken.abid,
-                apitoken.token_redacted[:64],
-                apitoken.expires,
-            )
-            for apitoken in obj.apitoken_set.order_by('-modified_at')[:10]
-        ) + f'<br/><a href="/admin/api/apitoken/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
-
-    @admin.display(description='API Outbound Webhooks')
-    def outboundwebhook_set(self, obj):
-        total_count = obj.outboundwebhook_set.count()
-        return mark_safe('<br/>'.join(
-            format_html(
-                '<code><a href="/admin/api/outboundwebhook/{}/change"><b>[{}]</b></a></code> {} -> {}',
-                outboundwebhook.pk,
-                outboundwebhook.abid,
-                outboundwebhook.referenced_model,
-                outboundwebhook.endpoint,
-            )
-            for outboundwebhook in obj.outboundwebhook_set.order_by('-modified_at')[:10]
-        ) + f'<br/><a href="/admin/api/outboundwebhook/?created_by__id__exact={obj.pk}">{total_count} total records...<a>')
-
-
-
-
-@abx.hookimpl
-def register_admin(admin_site):
-    admin_site.register(get_user_model(), CustomUserAdmin)
@@ -2,19 +2,27 @@ __package__ = 'archivebox.core'

 from django.apps import AppConfig

-import archivebox
-

 class CoreConfig(AppConfig):
    name = 'core'

    def ready(self):
-        """Register the archivebox.core.admin_site as the main django admin site"""
-        from django.conf import settings
-        archivebox.pm.hook.ready(settings=settings)
-        
-        from core.admin_site import register_admin_site
-        register_admin_site()
-        
+        # register our custom admin as the primary django admin
+        from django.contrib import admin
+        from django.contrib.admin import sites
+        from core.admin import archivebox_admin
+
+        admin.site = archivebox_admin
+        sites.site = archivebox_admin


+        # register signal handlers
+        from .auth import register_signals
+
+        register_signals()
+
+
+
+# from django.contrib.admin.apps import AdminConfig
+# class CoreAdminConfig(AdminConfig):
+#     default_site = "core.admin.get_admin_site"
@@ -1,26 +0,0 @@
-"""
-WSGI config for archivebox project.
-
-It exposes the WSGI callable as a module-level variable named ``application``.
-
-For more information on this file, see
-https://docs.djangoproject.com/en/2.1/howto/deployment/wsgi/
-"""
-
-from archivebox.config.django import setup_django
-
-setup_django(in_memory_db=False, check_db=True)
-
-
-from django.core.asgi import get_asgi_application
-from channels.routing import ProtocolTypeRouter
-
-
-django_asgi_app = get_asgi_application()
-
-application = ProtocolTypeRouter(
-    {
-        "http": django_asgi_app,
-        # Just HTTP for now. (We can add other protocols later.)
-    }
-)
@@ -0,0 +1,14 @@
+__package__ = 'archivebox.core'
+
+
+from ..config import (
+    LDAP
+)
+
+def register_signals():
+
+    if LDAP:
+        import django_auth_ldap.backend
+        from .auth_ldap import create_user
+
+        django_auth_ldap.backend.populate_user.connect(create_user)
@@ -0,0 +1,10 @@
+from ..config import (
+    LDAP_CREATE_SUPERUSER
+)
+
+def create_user(sender, user=None, ldap_user=None, **kwargs):
+    if not user.id and LDAP_CREATE_SUPERUSER:
+        user.is_superuser = True
+
+    user.is_staff = True
+    print(f'[!] WARNING: Creating new user {user} based on LDAP user {ldap_user} (is_staff={user.is_staff}, is_superuser={user.is_superuser})')
@@ -2,7 +2,7 @@ __package__ = 'archivebox.core'

 from django import forms

-from archivebox.misc.util import URL_REGEX
+from ..util import URL_REGEX
 from ..parsers import PARSERS
 from taggit.utils import edit_string_for_tags, parse_tags
Author	SHA1	Message	Date
Nick Sweeting	726bcabd82	fix iterator missing chunk_size	2024-08-26 22:18:10 -07:00
Nick Sweeting	d18418cc22	fix ArchiveResult.output_summary admin formatting	2024-08-26 22:17:57 -07:00