fix iterator missing chunk_size

fix ArchiveResult.output_summary admin formatting
2024-08-26 22:18:10 -07:00 · 2024-08-26 22:17:57 -07:00
895 changed files with 33817 additions and 148925 deletions
@@ -1,63 +1,24 @@
 .DS_Store
 ._*
-/.heartbeat.json
-/package.json
-/package-lock.json
-__pycache__
-**/__pycache__
 *.pyc
-*.pyo
-*.py[cod]
-*$py.class
+__pycache__/
 .mypy_cache/
 .pytest_cache/
-.ruff_cache/
-.uv-cache/
-.cache/
-.cache*/
 .github/
 .pdm-build/
-.pdm-build/**
 .pdm-python
 .eggs/
-*.egg-info/
 .git/
-!.git/
-.git/*
-.vscode/
 !.git/HEAD
-!.git/packed-refs
-!.git/refs/
-!.git/refs/heads/
-!.git/refs/heads/**
+!.git/refs/heads/*

 venv/
 .venv/
 .venv-old/
-.docker_venv/
 .docker-venv/
 node_modules/
-abx-dl/
-abxpkg/
-abx-plugins/
-abxbus/
 chrome/
 chromeprofile/
-chrome_profile/
-lib/
-out/
-tests/out/
-users/
-archive/
-crawls/
-snapshots/
-logs/
-archivebox-docker-smoke*/
-archivebox-compose-smoke*/
-docker-test/
-docker-test*/
-core
-*.core

 pdm.dev.lock
 pdm.lock
@@ -68,37 +29,11 @@ dist/
 brew_dist/
 deb_dist/
 pip_dist/
-cache/
-caches/
 assets/
 docker/
-website/
-typings/

-# The runtime COPY needs ArchiveBox's package and bin/ entrypoint, not the
-# development UI screenshots or test trees. PNGs are already compressed, so
-# including them creates a large image layer that later cleanup cannot remove.
-publicsite/
-old/
-test/
-tests/
-archivebox/tests/
-
-tmp/
-.tmp/
 data/
 data*/
-
-personas/
-sources/
 output/
 index.sqlite3
 index.sqlite3-wal
-queue.sqlite3
-*.sqlite*
-data.*
-.archivebox_id
-ArchiveBox.conf
-*.stdout
-*.stderr
-*.log
@@ -0,0 +1,2 @@
+**/*.lock
+**/*-lock.json
@@ -1,26 +0,0 @@
-# Read the Docs config for https://docs.archivebox.io
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
-
-version: 2
-
-submodules:
-  include: all
-  recursive: true
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.12"
-    #nodejs: "20"     # not needed unless we need the full archivebox to run while building docs for some reason
-
-sphinx:
-  configuration: docs/conf.py
-
-formats:
-  - pdf
-  - epub
-
-# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-python:
-  install:
-    - requirements: docs/requirements.txt
@@ -1,2 +1,2 @@
 github: ["ArchiveBox", "pirate"]
-custom: ["https://donate.archivebox.io", "https://swag.archivebox.io"]
+custom: ["https://donate.archivebox.io", "https://paypal.me/NicholasSweeting"]
@@ -1,197 +0,0 @@
-name: 🐞 Bug report
-description: Report a bug or error you encountered in ArchiveBox
-title: "Bug: ..."
-assignees:
-  - pirate
-type: 'Bug'
-body:
-  - type: markdown
-    attributes:
-      value: |
-        *Please note:* it is normal to see errors occasionally for some extractors on some URLs (not every extractor will work on every type of page).  
-        Please report archiving errors if you are seeing them *consistently across many URLs* or if they are *preventing you from using ArchiveBox*.
-
-  - type: textarea
-    id: description
-    attributes:
-      label: Provide a screenshot and describe the bug
-      description: |
-        Attach a screenshot and describe what the issue is, what you expected to happen, and if relevant, the *URLs you were trying to archive*.  
-      placeholder: |
-        Got a bunch of 'singlefile was unable to archive this page' errors when trying to archive URLs from this site: https://example.com/xyz ...
-        I also tried to archive the same URLs using `singlefile` directly and some of them worked but not all of them. etc. ...
-    validations:
-      required: true
-
-  - type: textarea
-    id: steps_to_reproduce
-    attributes:
-      label: Steps to reproduce
-      description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
-      render: markdown
-      placeholder: |
-        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
-        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
-        3. Typed 'https://example.com/xyz' into the 'Add URL' input field
-        4. Clicked the 'Add+' button
-        5. Got a 500 error and saw the errors below in terminal
-    validations:
-      required: true
-
-  - type: textarea
-    id: logs
-    attributes:
-      label: Logs or errors
-      description: "Paste any terminal output, logs, or errors (check `data/logs/errors.log` as well)."
-      placeholder: |
-        ╭─────────────────────────────────────────────────────────────────────────────────────────────────────────╮
-        │ [2024-11-02 19:54:28] ArchiveBox v0.8.6rc0: archivebox add https://example.com#1234567                  │
-        ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯
-        
-        [+] [2024-11-02 19:54:29] Adding 1 links to index (crawl depth=0)...
-            > Saved verbatim input to sources/1730577269-import.txt
-            > Parsed 1 URLs from input (Generic TXT)
-            ...
-      render: shell
-    validations:
-      required: false
-
-  - type: textarea
-    id: version
-    attributes:
-      label: ArchiveBox Version
-      description: |
-        **REQUIRED:** Run the `archivebox version` command inside your collection dir and paste the *full output* here (*not just the version number*).  
-        For Docker Compose run: `docker compose run archivebox version`  
-        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
-      render: shell
-      placeholder: |
-        0.8.6
-        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
-        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
-        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
-        
-        Binary Dependencies:
-        √  python     3.14.0     venv_pip   ~/.venv/bin/python
-        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
-        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
-        ...
-    validations:
-      required: true
-
-  - type: dropdown
-    id: install_method
-    validations:
-      required: true
-    attributes:
-      label: How did you install the version of ArchiveBox you are using?
-      multiple: false
-      options:
-        - pip
-        - apt
-        - brew
-        - nix
-        - Docker (or Podman/LXC/K8s/TrueNAS/Proxmox/etc)
-        - Other
-
-  - type: dropdown
-    id: operating_system
-    validations:
-      required: true
-    attributes:
-      label: What operating system are you running on?
-      description: |
-        Please note we are *unable to provide support for Windows users* unless you are using [Docker on Windows](https://github.com/ArchiveBox/archivebox#:~:text=windows%20without%20docker).
-      multiple: false
-      options:
-        - Linux (Ubuntu/Debian/Arch/Alpine/etc.)
-        - macOS (including Docker on macOS)
-        - BSD (FreeBSD/OpenBSD/NetBSD/etc.)
-        - Windows (including WSL, WSL2, Docker Desktop on Windows)
-        - Other
-
-  - type: checkboxes
-    id: filesystem
-    attributes:
-      label: What type of drive are you using to store your ArchiveBox data?
-      description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` OR `data/archive`?
-      options:
-        - label: "some of `data/` is on a local SSD or NVMe drive"
-          required: false
-        - label: "some of `data/` is on a spinning hard drive or external USB drive"
-          required: false
-        - label: "some of `data/` is on a network mount (e.g. NFS/SMB/Ceph/GlusterFS/etc.)"
-          required: false
-        - label: "some of `data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/Google Drive/Dropbox/etc.)"
-          required: false
-  
-
-  - type: textarea
-    id: docker_compose_yml
-    attributes:
-      label: Docker Compose Configuration
-      description: "If using Docker Compose, please share your full `docker-compose.yml` file. If using plain Docker, paste the `docker run ...` command you use."
-      placeholder: |
-        services:
-            archivebox:
-                image: archivebox/archivebox:latest
-                ports:
-                    - 8000:8000
-                volumes:
-                    - ./data:/data
-                environment:
-                    - ADMIN_USERNAME=admin
-                    - ADMIN_PASSWORD=****<redact any passwords>****
-                    - ALLOWED_HOSTS=*
-                    - CSRF_TRUSTED_ORIGINS=https://archivebox.example.com
-                    - PUBLIC_INDEX=True
-                    - PUBLIC_SNAPSHOTS=True
-                    - PUBLIC_ADD_VIEW=False
-                    ...
-
-            archivebox_scheduler:
-                image: archivebox/archivebox:latest
-                command: schedule --foreground --update --every=day
-                environment:
-                    ...
-      
-          ...
-      render: shell
-    validations:
-      required: false
-
-  - type: textarea
-    id: configuration
-    attributes:
-      label: ArchiveBox Configuration
-      description: "Please share your full `data/ArchiveBox.conf` file here."
-      render: shell
-      placeholder: |
-        [SERVER_CONFIG]
-        SECRET_KEY = "*********<redact any secrets/passwords>************"
-
-        WGET_RESTRICT_FILE_NAMES=windows
-        USE_SYSTEM_WGET=true
-        CHECK_SSL_VALIDITY=false
-        ...
-    validations:
-      required: false
-
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying anything suggested here, **we are not responsible for data loss**.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
-          - 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
-          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
-        
@@ -1,124 +0,0 @@
-name: 💡 Feature or enhancement request
-description: Suggest an idea or improvement for this project
-title: "Feature Request: ..."
-assignees:
-  - pirate
-type: 'Enhancement'
-labels: 'status: idea phase'
-body:
-  - type: dropdown
-    id: suggestion_type
-    validations:
-      required: true
-    attributes:
-      label: "What type of suggestion are you making?"
-      multiple: false
-      options:
-        - New extractor / type of content to save
-        - Proposing a new feature
-        - Modification of existing behavior
-        - Web UI or UX design improvement
-
-  - type: textarea
-    id: current_problem
-    attributes:
-      label: "What is the problem that your feature request solves?"
-      description: |
-        Describe the problem or need that your feature request solves, feel free to include any screenshots or examples.
-      placeholder: |
-        e.g. I need to be able to archive spanish and french subtitle files from a particular movie site https://example.com/somevideos that's going down soon.
-    validations:
-      required: true
-
-  - type: textarea
-    id: proposed_solution
-    attributes:
-      label: "What is your proposed solution?"
-      description: |
-        Describe the ideal specific solution you'd want, *and whether it fits into any broader scope of changes*.
-      placeholder: |
-        e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
-        The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
-    validations:
-      required: true
-
-  - type: textarea
-    id: workarounds_tried
-    attributes:
-      label: "What hacks or alternative solutions have you tried to solve the problem?"
-      description: |
-        A description of any alternative approaches, workarounds, or other solutions you've considered to fix the problem.
-      placeholder: |
-        e.g. I wait for archivebox to finish archiving the page, then I manually run `yt-dlp --subs <URL>` inside 
-        the `data/archive/<timestamp>/` directory to download the subtitle files and add them to the snapshot folder.
-    validations:
-      required: true
-
-  - type: textarea
-    id: version
-    attributes:
-      label: Share the entire output of the `archivebox version` command for the current version you are using.
-      description: |
-        DO NOT JUST ENTER "the latest version" OR YOUR ISSUE WILL BE CLOSED.  
-        We need to know what version of ArchiveBox and what feature flags you're currently running with in order to contextualize your feature request.  
-        Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in your specific environment.
-          
-        Run the `archivebox version` command inside your current collection dir and paste the *full output* here (*not just the version number*).  
-        For Docker Compose run: `docker compose run archivebox version`  
-        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
-      render: shell
-      placeholder: |
-        0.8.6
-        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
-        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
-        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
-        
-        Binary Dependencies:
-        √  python     3.14.0     venv_pip   ~/.venv/bin/python
-        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
-        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
-        ...
-    validations:
-      required: true
-
-  - type: checkboxes
-    id: priority
-    attributes:
-      label: "How badly do you want this new feature?"
-      options:
-        - label: "It's an urgent deal-breaker, I can't live without it"
-          required: false
-        - label: "It's important to add it in the near-mid term future"
-          required: false
-        - label: "It would be nice to have eventually"
-          required: false
-        - label: "I'm willing to [start a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
-          required: false
-        - label: "I have [donated money](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations) to go towards fixing this issue"
-          required: false
-  
-  - type: checkboxes
-    id: satisfaction_survey
-    attributes:
-      label: Mini Survey
-      description: How do you like ArchiveBox so far?
-      options:
-        - label: "I like ArchiveBox so far / would recommend it to a friend"
-          required: false
-        - label: "I've had a lot of difficulty getting ArchiveBox set up"
-          required: false
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying any workarounds or BETAs suggested here, **we are not responsible for data loss**.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
-          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
@@ -1,51 +0,0 @@
-name: 📑 Documentation improvement
-description: Submit an idea or correction for the Wiki documentation
-title: "Documentation: ..."
-labels: 'touches: docs'
-type: 'Enhancement'
-assignees:
-  - pirate
-body:
-  - type: markdown
-    attributes:
-      value: |
-        If you prefer, you can submit a [Pull Request](https://github.com/ArchiveBox/docs) on https://github.com/ArchiveBox/docs to edit the docs directly instead.
-
-  - type: input
-    id: page_url
-    validations:
-      required: true
-    attributes:
-      label: "What is the URL of the page you'd like to see improved?"
-      placeholder: e.g. https://github.com/ArchiveBox/docs/wiki/Install
-
-  - type: input
-    id: section_title
-    validations:
-      required: true
-    attributes:
-      label: "What is the title of the relevant section?"
-      placeholder: e.g. Option B. Automatic Setup Script
-
-  - type: textarea
-    id: suggested_edit
-    attributes:
-      label: "What is the suggested edit?"
-      placeholder: |
-        e.g. Please document how to run the automatic setup script for ArchiveBox on TempleOS.  
-        Attach images, screenshots, code snippets, etc. anything you think would help.
-    validations:
-      required: true
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to address issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
-          - ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)
@@ -0,0 +1,46 @@
+---
+name: 🐞 Bug report
+about: Create a report to help us improve
+title: 'Bug: ...'
+labels: 'bug'
+assignees: ''
+
+---
+
+<!--
+Please fill out the following information, 
+feel free to delete sections if they're not applicable 
+or if long issue templates annoy you.
+(the only required section is the version information)
+-->
+
+#### Describe the bug
+<!--
+A description of what the bug is, 
+what you expected to happen, 
+and any relevant context about issue.
+-->
+
+#### Steps to reproduce
+<!--
+For example:
+1. Ran ArchiveBox with the following config '...'
+2. Saw this output during archiving '....'
+3. UI didn't show the thing I was expecting '....'
+-->
+
+#### Screenshots or log output
+
+<!--
+If applicable, post any relevant screenshots or copy/pasted terminal output from ArchiveBox.
+If you're reporting a parsing / importing error, **you must paste a copy of your redacted import file here**.
+-->
+
+#### ArchiveBox version
+
+<!-- Run the `archivebox version` command locally then copy paste the result here: -->
+```logs
+replace this line with the *full*, unshortened output of running `archivebox version`
+```
+<!-- Tickets without full version info will closed until it is provided,
+we need the full output here to help you solve your issue -->
@@ -1,8 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: ❓ Ask a question or start a discussion
-    url: https://github.com/ArchiveBox/ArchiveBox/discussions
-    about: "Ask a question, get support, or start a design discussion (to report a problem please use '🐞 Bug report' instead)"
-  - name: 💬 Chat with the dev team & community on Zulip
-    url: https://zulip.archivebox.io
-    about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
@@ -0,0 +1,17 @@
+---
+name: 📑 Documentation change
+about: Submit a suggestion for the Wiki documentation
+title: 'Documentation: Improvement request ...'
+labels: ''
+assignees: ''
+
+---
+<!-- If you perfer, you can make a PR to https://github.com/ArchiveBox/docs instead of opening an issue -->
+
+## Wiki Page URL
+<!-- e.g. https://github.com/pirate/ArchiveBox/wiki/Configuration#use_color -->
+
+
+## Suggested Edit
+<!-- e.g. Please add more example usages, or please fix `xyz` typo to be `abc`. -->
+
@@ -0,0 +1,50 @@
+---
+name: 💡 Feature request
+about: Suggest an idea for this project
+title: 'Feature Request: ...'
+labels: 'changes: behavior,status: idea phase'
+assignees: ''
+
+---
+
+<!--
+Please fill out the following information, 
+feel free to delete sections if they're not applicable 
+or if long issue templates annoy you :)
+-->
+
+## Type 
+
+ - [ ] General question or discussion
+ - [ ] Propose a brand new feature
+ - [ ] Request modification of existing behavior or design
+
+## What is the problem that your feature request solves
+<!--
+e.g. I need to be able to archive spanish and french subtitle files  
+from a particular <example.com> movie site that's going down soon.
+-->
+
+## Describe the ideal specific solution you'd want, and whether it fits into any broader scope of changes
+<!--
+e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
+The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
+-->
+
+## What hacks or alternative solutions have you tried to solve the problem?
+<!--
+A clear and concise description of any alternative solutions, 
+workarounds, or other software you've considered using to fix the problem.
+-->
+
+## How badly do you want this new feature?
+
+ - [ ] It's an urgent deal-breaker, I can't live without it
+ - [ ] It's important to add it in the near-mid term future
+ - [ ] It would be nice to have eventually
+
+---
+
+ - [ ] I'm willing to contribute [dev time](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) / [money](https://github.com/sponsors/pirate) to fix this issue
+ - [ ] I like ArchiveBox so far / would recommend it to a friend
+ - [ ] I've had a lot of difficulty getting ArchiveBox set up
@@ -0,0 +1,9 @@
+---
+name: 💬 Question, discussion, or support request
+about: Start a discussion or ask a question about ArchiveBox
+title: 'Question: ...'
+labels: ''
+assignees: ''
+
+---
+
@@ -1,68 +0,0 @@
-# Security Policy
-
---
-
-## Security Documentation
-
-Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
-
- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
- https://github.com/ArchiveBox/ArchiveBox#archive-layout
- https://github.com/ArchiveBox/ArchiveBox#archivebox-development
- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
-
---
-
-## Reporting a Vulnerability
-
-~We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.~ We no longer offer CVEs or accept private reporting, 95% of the reports we received were low quality AI slop, so now we just ask for normal Github Issues or PRs.
-
-You can contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp) if you have a truly urgent security report, but we do not issue public CVEs.
-
---
-
-## CVEs and Report Credits
-
-> [!IMPORTANT]
-> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**  
-> Reporters **will still receive credit in release notes** however.
-
-> [!TIP]
-> ~Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.~
-
-Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
-
---
-
-## Threat Model
-
-### All admin users are assumed to be trusted
-
-ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
-
- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
- non-logged in anonymous guest users
-
-Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
-
-### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
-
-Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
-
-Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
-
-### XSS, CSRF, CORS, CSP
-
-`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
-
-[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
-
-### SSRF / URL Validation
-
-Note archivebox is **designed to be able to archive any URL the server can reach**, including internal intranet URLs, private IPs, and localhost. Users are expected to configure `URL_ALLOWLIST`/`URL_DENYLIST` or proper firewall/ACL-level blocking at the network level if they need to limit what the server should see.
-It does a basic check to try and avoid recursively archiving snapshots hosted by the archivebox server itself, but it's not a strict security check, it's just to prevent wasted CPU/disk from recursively archiving snapshots in a browser that has the ArchiveBox Browser Extension turned on.
@@ -1,81 +0,0 @@
-{
-  "properties": {
-    "CC_BINARY": {
-      "default": "cc"
-    },
-    "LDAPSEARCH_BINARY": {
-      "default": "ldapsearch"
-    },
-    "INITDB_BINARY": {
-      "default": "initdb"
-    },
-    "PG_CTL_BINARY": {
-      "default": "pg_ctl"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{CC_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "build-essential"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{LDAPSEARCH_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapsearch",
-            "-VV"
-          ]
-        },
-        "apt": {
-          "install_args": [
-            "ldap-utils",
-            "python3-dev",
-            "python3-setuptools",
-            "libssl-dev",
-            "libldap2-dev",
-            "libsasl2-dev",
-            "zlib1g-dev",
-            "libatomic1"
-          ],
-          "version": [
-            "ldapsearch",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ],
-  "postgres_binaries": [
-    {
-      "name": "{INITDB_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "postgresql"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{PG_CTL_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "postgresql"
-          ]
-        }
-      }
-    }
-  ]
-}
@@ -1,13 +0,0 @@
-{
-  "properties": {
-    "BREW_BINARY": {
-      "default": "brew"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{BREW_BINARY}",
-      "binproviders": "env"
-    }
-  ]
-}
@@ -1,44 +0,0 @@
-{
-  "properties": {
-    "CLANG_BINARY": {
-      "default": "clang"
-    },
-    "LDAPVC_BINARY": {
-      "default": "ldapvc"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{CLANG_BINARY}",
-      "binproviders": "env,brew",
-      "overrides": {
-        "brew": {
-          "install_args": [
-            "llvm"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{LDAPVC_BINARY}",
-      "binproviders": "env,brew",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        },
-        "brew": {
-          "install_args": [
-            "openldap"
-          ],
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ]
-}
@@ -1,21 +0,0 @@
-{
-  "properties": {
-    "LDAPVC_BINARY": {
-      "default": "ldapvc"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{LDAPVC_BINARY}",
-      "binproviders": "env",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ]
-}
@@ -1,345 +0,0 @@
-{
-  "properties": {
-    "CURL_BINARY": {
-      "default": "curl"
-    },
-    "BASH_BINARY": {
-      "default": "bash"
-    },
-    "CHMOD_BINARY": {
-      "default": "chmod"
-    },
-    "DOCKER_BINARY": {
-      "default": "docker"
-    },
-    "GIT_BINARY": {
-      "default": "git"
-    },
-    "GREP_BINARY": {
-      "default": "grep"
-    },
-    "GH_BINARY": {
-      "default": "gh"
-    },
-    "EXPORTFS_BINARY": {
-      "default": "exportfs"
-    },
-    "JQ_BINARY": {
-      "default": "jq"
-    },
-    "MKDIR_BINARY": {
-      "default": "mkdir"
-    },
-    "MOUNT_CIFS_BINARY": {
-      "default": "mount.cifs"
-    },
-    "MOUNT_NFS_BINARY": {
-      "default": "mount.nfs"
-    },
-    "NODE_BINARY": {
-      "default": "node"
-    },
-    "PS_BINARY": {
-      "default": "ps"
-    },
-    "PYTHON_BINARY": {
-      "default": "python"
-    },
-    "PV_BINARY": {
-      "default": "pv"
-    },
-    "SSH_BINARY": {
-      "default": "ssh"
-    },
-    "SED_BINARY": {
-      "default": "sed"
-    },
-    "SMBD_BINARY": {
-      "default": "smbd"
-    },
-    "SUDO_BINARY": {
-      "default": "sudo"
-    },
-    "SYSTEMCTL_BINARY": {
-      "default": "systemctl"
-    },
-    "TAR_BINARY": {
-      "default": "tar"
-    },
-    "TREE_BINARY": {
-      "default": "tree"
-    },
-    "TR_BINARY": {
-      "default": "tr"
-    },
-    "RM_BINARY": {
-      "default": "rm"
-    },
-    "UMOUNT_BINARY": {
-      "default": "umount"
-    },
-    "UV_BINARY": {
-      "default": "uv"
-    },
-    "UNAME_BINARY": {
-      "default": "uname"
-    }
-  },
-  "ci_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{JQ_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{UV_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "bsd_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,pyinfra"
-    }
-  ],
-  "freebsd_binaries": [
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env,pyinfra"
-    }
-  ],
-  "docker_binaries": [
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UNAME_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "deploy_binaries": [
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SSH_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "docker_debug_binaries": [
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{PV_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{TAR_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{TREE_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "docker_validation_binaries": [
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GREP_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{MKDIR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{RM_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SED_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SSH_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{TR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UNAME_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "mount_validation_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{CHMOD_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{EXPORTFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "nfs-kernel-server"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{MKDIR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{MOUNT_CIFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "cifs-utils"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{MOUNT_NFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "nfs-common"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{SMBD_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "samba"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SYSTEMCTL_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UMOUNT_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "process_binaries": [
-    {
-      "name": "{PS_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "docs_binaries": [
-    {
-      "name": "{CURL_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "git_binaries": [
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "github_binaries": [
-    {
-      "name": "{GH_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "node_binaries": [
-    {
-      "name": "{NODE_BINARY}",
-      "binproviders": "env,npm",
-      "min_version": "22.23.1",
-      "overrides": {
-        "npm": {
-          "install_args": [
-            "node@22.23.1"
-          ],
-          "postinstall_scripts": true
-        }
-      }
-    }
-  ],
-  "release_binaries": [
-    {
-      "name": "{UV_BINARY}",
-      "binproviders": "env,pip,brew"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{JQ_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{CURL_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ]
-}
@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-"""Discover every ArchiveBox test file and expose it as one CI job."""
-
-import json
-from pathlib import Path
-
-
-def main() -> None:
-    root = Path.cwd().resolve()
-    archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
-
-    if not archivebox_tests:
-        raise SystemExit("No ArchiveBox tests discovered")
-
-    matrix: list[dict[str, object]] = []
-    for path in archivebox_tests:
-        test_path = path.relative_to(root).as_posix()
-        matrix.append(
-            {
-                "name": f"main/{path.stem}",
-                "paths": [test_path],
-                "paths_arg": test_path,
-                "extra": "ldap" if test_path.endswith("/test_auth_ldap.py") else "",
-                "count": 1,
-            },
-        )
-
-    discovered_paths = [path for entry in matrix for path in entry["paths"]]
-    if len(discovered_paths) != len(set(discovered_paths)):
-        raise SystemExit("Tests were not discovered exactly once")
-    if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
-        raise SystemExit("Discovered test shard coverage does not match test files")
-
-    print(f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs")
-    print(json.dumps(matrix, separators=(",", ":")))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,92 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-    branches: [dev, main]
-  push:
-    branches: [dev, main]
-  workflow_dispatch:
-
-permissions:
-  actions: read
-  contents: read
-  packages: write
-  security-events: write
-
-concurrency:
-  group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }}
-  cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
-
-jobs:
-  lint:
-    name: Linters
-    uses: ./.github/workflows/lint.yml
-    secrets: inherit
-
-  install-cli-platform:
-    name: Install and CLI platform compatibility
-    uses: ./.github/workflows/test.yml
-    secrets: inherit
-
-  tests:
-    name: Discovered test matrix
-    uses: ./.github/workflows/test-parallel.yml
-    secrets: inherit
-
-  documentation:
-    name: Documentation and root tests
-    uses: ./.github/workflows/docs.yml
-    secrets: inherit
-
-  codeql:
-    name: CodeQL
-    uses: ./.github/workflows/codeql.yml
-    secrets: inherit
-
-  python-artifacts:
-    name: Tested Python artifacts
-    uses: ./.github/workflows/pip.yml
-    secrets: inherit
-
-  docker-digests:
-    name: Tested Docker images
-    uses: ./.github/workflows/docker.yml
-    with:
-      push_digests: false
-    secrets: inherit
-
-  required:
-    name: All required CI lanes
-    if: ${{ always() }}
-    needs:
-      - lint
-      - install-cli-platform
-      - tests
-      - documentation
-      - codeql
-      - python-artifacts
-      - docker-digests
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Verify every required lane succeeded
-        env:
-          LINT_RESULT: ${{ needs.lint.result }}
-          INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
-          TESTS_RESULT: ${{ needs.tests.result }}
-          DOCUMENTATION_RESULT: ${{ needs.documentation.result }}
-          CODEQL_RESULT: ${{ needs.codeql.result }}
-          PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
-          DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
-        run: |
-          for result in \
-            "$LINT_RESULT" \
-            "$INSTALL_CLI_PLATFORM_RESULT" \
-            "$TESTS_RESULT" \
-            "$DOCUMENTATION_RESULT" \
-            "$CODEQL_RESULT" \
-            "$PYTHON_ARTIFACTS_RESULT" \
-            "$DOCKER_DIGESTS_RESULT"
-          do
-            [[ "$result" == success ]]
-          done
-          echo 'All required CI lanes and tested release artifacts succeeded.'
@@ -1,40 +0,0 @@
-name: Claude Code
-
-on:
-  issue_comment:
-    types: [created]
-  pull_request_review_comment:
-    types: [created]
-  issues:
-    types: [opened, assigned]
-  pull_request_review:
-    types: [submitted]
-
-jobs:
-  claude:
-    if: |
-      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
-      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: read
-      pull-requests: write
-      issues: write
-      id-token: write
-      actions: read
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Run Claude Code
-        id: claude
-        uses: anthropics/claude-code-action@b76a0776ae74036e77cd11018083743453d7ad35 # v1
-        with:
-          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
-          additional_permissions: |
-            actions: read
-          claude_args: '--allowed-tools Bash(gh pr:*)'
@@ -1,21 +1,42 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
 name: "CodeQL"

 on:
-  workflow_call:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
  schedule:
    - cron: '33 17 * * 6'

-concurrency:
-  group: codeql-${{ github.repository }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
  analyze:
    name: Analyze (${{ matrix.language }})
-    runs-on: ubuntu-24.04
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
    permissions:
+      # required for all workflows
      security-events: write
+
+      # required to fetch internal or private CodeQL packs
      packages: read
+
+      # only required for workflows in private repositories
      actions: read
      contents: read

@@ -25,18 +46,47 @@ jobs:
        include:
        - language: python
          build-mode: none
-        - language: javascript-typescript
-          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
    steps:
    - name: Checkout repository
-      uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      uses: actions/checkout@v4

+    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
-      uses: github/codeql-action/init@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
+      uses: github/codeql-action/init@v3
      with:
        languages: ${{ matrix.language }}
        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
+      uses: github/codeql-action/analyze@v3
      with:
        category: "/language:${{matrix.language}}"
@@ -0,0 +1,64 @@
+name: Build Debian package
+
+on:
+  workflow_dispatch:
+  push:
+
+env:
+  DEB_BUILD_OPTIONS: nocheck
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Install packaging dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y \
+            python3 python3-dev python3-pip python3-venv python3-all \
+            dh-python debhelper devscripts dput software-properties-common \
+            python3-distutils python3-setuptools python3-wheel python3-stdeb
+
+      # - name: Build Debian/Apt sdist_dsc
+      #   run: |
+      #     ./bin/build_pip.sh
+
+      # - name: Check ArchiveBox version
+      #   run: |
+      #     # must create dir needed for snaps to run as non-root on github actions
+      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
+      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
+      #     archivebox --version
+      #     archivebox init --setup
+
+      # - name: Add some links to test
+      #   run: |
+      #     cd "${{ github.workspace }}/data"
+      #     archivebox add 'https://example.com'
+      #     archivebox status
+
+      # - name: Commit built package
+      #   run: |
+      #     cd deb_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Debian package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/debian-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: deb_dist
+
+      # - name: Push build to Launchpad PPA
+      #   run: |
+      #     debsign -k "$PGP_KEY_ID" "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
+      #     dput archivebox "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
@@ -1,78 +0,0 @@
-name: Deploy Publicsite to GitHub Pages
-
-on:
-  push:
-    branches:
-      - dev
-    paths:
-      - publicsite/**
-      - archivebox/**
-      - bin/collect_ui_screenshots.*
-      - bin/generate_ui_screenshot_gallery.py
-      - bin/take_screenshot.js
-      - pyproject.toml
-      - uv.lock
-      - .github/workflows/deploy-publicsite.yml
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-concurrency:
-  group: ${{ github.event_name == 'workflow_dispatch' && format('github-pages-publicsite-manual-{0}', github.run_id) || 'github-pages-publicsite' }}
-  cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
-
-jobs:
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Setup Python and uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: false
-
-      - name: Install ArchiveBox and screenshot dependencies
-        run: uv sync --no-cache --frozen
-
-      - name: Build UI screenshot gallery
-        run: bin/collect_ui_screenshots.sh
-        env:
-          BASE_URL: http://archivebox.localhost:8000
-
-      - name: Verify gallery was generated from this exact revision
-        run: |
-          uv run --no-cache --no-sync python - <<'PY'
-          import json
-          import os
-          import tomllib
-          from pathlib import Path
-
-          provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
-          version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
-          assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
-          assert provenance["version"] == version, provenance
-          assert provenance["capture_count"] >= 3 * 35, provenance
-          assert len(provenance["files"]) == provenance["capture_count"], provenance
-          PY
-
-      - name: Setup Pages
-        uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
-
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
-        with:
-          path: ./publicsite
-
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
@@ -1,115 +1,37 @@
 name: Build Docker image

 on:
-  workflow_call:
-    inputs:
-      push_digests:
-        description: Push validated per-platform images by digest for the release workflow
-        required: false
-        default: false
-        type: boolean
-      full_tests:
-        description: Run storage, CLI, and size validation in addition to release smoke checks
-        required: false
-        default: true
-        type: boolean
+  workflow_dispatch:
+  push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'
+  # pull_request:

 env:
-  DOCKERHUB_IMAGE: archivebox/archivebox
-  GHCR_IMAGE: ghcr.io/archivebox/archivebox
-
-permissions:
-  actions: read
-  contents: read
-  packages: write
+  DOCKER_IMAGE: archivebox-ci

 jobs:
-  build:
-    name: build ${{ matrix.platform }}
-    runs-on: ${{ matrix.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: linux/amd64
-            runner: ubuntu-24.04
-            artifact_name: digest-linux-amd64
-            local_tag: archivebox-ci:amd64
-          - platform: linux/arm64
-            runner: ubuntu-24.04-arm
-            artifact_name: digest-linux-arm64
-            local_tag: archivebox-ci:arm64
+  buildx:
+    runs-on: ubuntu-latest
    steps:
      - name: Checkout
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/checkout@v4
        with:
-          python-version: "3.13"
+          submodules: true
+          fetch-depth: 1

-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Resolve Docker through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python - <<'PY'
-          import tomllib
-          from pathlib import Path
-
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          matches = [package for package in packages if package["name"] == "abxpkg"]
-          if len(matches) != 1:
-              raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}")
-          wheel = matches[0]["wheels"][0]
-          print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))
-          PY
-          )"
-          test -n "$ABXPKG_SPEC"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_SPEC=$ABXPKG_SPEC" >> "$GITHUB_ENV"
-          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-
-          docker_env="$(
-            uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries"
-          )"
-          JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$docker_env" >> "$GITHUB_ENV"
-
-          docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
-          UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv"
-          {
-            echo "DOCKER_BINARY=$docker_binary"
-            echo "JQ_BINARY=$JQ_BINARY"
-            echo "UV_BINARY=$UV_BINARY"
-          } >> "$GITHUB_ENV"
-          for binary in "$docker_binary" "$JQ_BINARY" "$UV_BINARY"; do
-            test -L "$binary"
-            test -x "$binary"
-          done
-          "$docker_binary" --version
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        id: buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+        uses: docker/setup-buildx-action@v3
        with:
+          version: latest
          install: true
+          platforms: linux/amd64,linux/arm64

      - name: Builder instance name
        run: echo ${{ steps.buildx.outputs.name }}
@@ -117,398 +39,68 @@ jobs:
      - name: Available platforms
        run: echo ${{ steps.buildx.outputs.platforms }}

-      - name: Verify published abx-dl image
-        id: abx_dl_image
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABX_DL_VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import tomllib
-          from pathlib import Path
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-

-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          matches = [package["version"] for package in packages if package["name"] == "abx-dl"]
-          if len(matches) != 1:
-              raise SystemExit(f"Expected one locked abx-dl package, found {len(matches)}")
-          print(matches[0])
-          PY
-          )"
-          test -n "$ABX_DL_VERSION"
-          ABX_DL_REFERENCE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
-          INSPECT_OUTPUT="$("$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_REFERENCE")"
-          printf '%s\n' "$INSPECT_OUTPUT"
-          ABX_DL_DIGEST="$(INSPECT_OUTPUT="$INSPECT_OUTPUT" "$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import os
-
-          digest = next(
-              (fields[1] for line in os.environ["INSPECT_OUTPUT"].splitlines() if (fields := line.split()) and fields[0] == "Digest:"),
-              "",
-          )
-          print(digest)
-          PY
-          )"
-          [[ "$ABX_DL_DIGEST" == sha256:* ]]
-          ABX_DL_IMAGE="${ABX_DL_REFERENCE%%@*}@${ABX_DL_DIGEST}"
-          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        if: inputs.push_digests
+      - name: Docker Login
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        if: inputs.push_digests
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Collect Docker labels
+      - name: Collect Docker tags
+        # https://github.com/docker/metadata-action
        id: docker_meta
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import tomllib
-          from pathlib import Path
+        uses: docker/metadata-action@v5
+        with:
+          images: archivebox/archivebox,nikisweeting/archivebox
+          tags: |
+              # :stable
+              type=ref,event=branch
+              # :0.7.3
+              type=semver,pattern={{version}}
+              # :0.7
+              type=semver,pattern={{major}}.{{minor}}
+              # :sha-463ea54
+              type=sha
+              # :latest
+              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}

-          print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])
-          PY
-          )"
-          test -n "$VERSION"
-
-          {
-            echo 'labels<<EOF'
-            echo "org.opencontainers.image.version=${VERSION}"
-            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
-            echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
-            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
-            echo 'EOF'
-            echo "version=${VERSION}"
-          } >> "$GITHUB_OUTPUT"
-
-          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
-
-      - name: Build and push digest
+      - name: Build and push
        id: docker_build
-        if: inputs.push_digests
-        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
+        uses: docker/build-push-action@v5
        with:
          context: ./
          file: ./Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-          build-args: |
-            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
-          pull: true
-          platforms: ${{ matrix.platform }}
-          # Keep the released image as the one cache authority. A second GHA
-          # cache can hold gzip variants of identical layers, and BuildKit may
-          # prefer those over :dev's zstd blobs. That counter-intuitively makes
-          # a cache hit larger than the released image. :dev advances only
-          # after both architectures pass the release checks.
-          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
-          # Inline metadata reuses the released compressed blobs directly;
-          # force-compression would recompress the whole ~740 MiB image. The
-          # compressed-size gate below remains the regression guard.
-          cache-to: |
-            ${{ inputs.push_digests && 'type=inline' || '' }}
-          outputs: |
-            ${{ inputs.full_tests && format('type=oci,dest={0}/archivebox-image.tar,compression=zstd,compression-level=3', runner.temp) || '' }}
-            ${{ inputs.push_digests && format('type=image,"name={0},{1}",push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=3', env.DOCKERHUB_IMAGE, env.GHCR_IMAGE) || '' }}
-
-      - name: Build pull request image
-        id: docker_build_pr
-        if: ${{ !inputs.push_digests }}
-        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
-        with:
-          context: ./
-          file: ./Dockerfile
-          builder: ${{ steps.buildx.outputs.name }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-          build-args: |
-            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
-          pull: true
-          platforms: ${{ matrix.platform }}
-          # Test the same layer lineage users receive. Reusing :dev avoids a
-          # second cache whose compression can drift from the released image.
-          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
-          outputs: |
-            type=docker,name=${{ matrix.local_tag }}
-            type=oci,dest=${{ runner.temp }}/archivebox-image.tar
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+          platforms: linux/amd64,linux/arm64

      - name: Image digest
-        if: inputs.push_digests
        run: echo ${{ steps.docker_build.outputs.digest }}

-      - name: Validate exact built image version and commit
-        if: ${{ !inputs.push_digests }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          IMAGE="${{ matrix.local_tag }}"
-          SHORT_SHA="${GITHUB_SHA::7}"
-
-          DATA_DIR="$(mktemp -d)"
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
-          ARCHIVEBOX_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version)"
-          printf '%s\n' "$ARCHIVEBOX_VERSION_OUTPUT"
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
-          IMAGE_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt)"
-          ARCHIVEBOX_VERSION_OUTPUT="$ARCHIVEBOX_VERSION_OUTPUT" IMAGE_VERSION_OUTPUT="$IMAGE_VERSION_OUTPUT" \
-            "$UV_BINARY" run --no-cache --no-project python - \
-            "ArchiveBox v${{ steps.docker_meta.outputs.version }}" \
-            "COMMIT_HASH=${SHORT_SHA}" \
-            "COMMIT_HASH=${GITHUB_SHA}" <<'PY'
-          import os
-          import sys
-
-          version_output = os.environ["ARCHIVEBOX_VERSION_OUTPUT"]
-          image_version_lines = os.environ["IMAGE_VERSION_OUTPUT"].splitlines()
-          if sys.argv[1] not in version_output:
-              raise SystemExit(f"Missing version marker: {sys.argv[1]}")
-          if sys.argv[2] not in version_output:
-              raise SystemExit(f"Missing short commit marker: {sys.argv[2]}")
-          if sys.argv[3] not in image_version_lines:
-              raise SystemExit(f"Missing exact commit marker: {sys.argv[3]}")
-          PY
-
-          "$DOCKER_BINARY" run --rm --user 911:911 --platform '${{ matrix.platform }}' --entrypoint /venv/bin/python3 "$IMAGE" -c \
-            'import json, pathlib, stat; cache_file = "/opt/archivebox/lib/env/derived.env"; raw = open(cache_file).read().strip(); records = json.loads(raw[len("ABXPKG_DERIVED_CACHE=\x27"):-1]); projections = [projection for record in records.values() for projection in record.get("request_exec_projections", {}).values()]; plans = [projection["validation"] for projection in projections]; fingerprints = [fingerprint for plan in plans for fingerprint in plan["fingerprint"]]; actual = lambda fingerprint: (lambda path, value: {"path": str(path.resolve(strict=False)), "size": value.st_size, "mtime_ns": value.st_mtime_ns, "mode": stat.S_IMODE(value.st_mode), "euid": value.st_uid})(pathlib.Path(fingerprint["path"]), pathlib.Path(fingerprint["path"]).stat()); mismatches = [(fingerprint, actual(fingerprint)) for fingerprint in fingerprints if fingerprint != actual(fingerprint)]; assert projections and {plan.get("euid") for plan in plans} == {911}; assert all(not record.get("script_exec_plans") for record in records.values()); assert "ABXPKG_TMP_CACHE_DIR" not in json.dumps(projections); assert not mismatches, mismatches[:3]'
-
-      - name: Validate compressed candidate size
-        if: inputs.push_digests
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          DIGEST='${{ steps.docker_build.outputs.digest }}'
-          [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
-          MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${DIGEST}")"
-          # Provenance wraps the real platform image and its attestation in an
-          # OCI index. Measure the image manifest, not the index/attestation.
-          IMAGE_DIGEST="$(MANIFEST="$MANIFEST" PLATFORM='${{ matrix.platform }}' "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); platform_os, platform_arch = os.environ["PLATFORM"].split("/")[:2]; print(next((item["digest"] for item in manifest.get("manifests", []) if item.get("platform", {}).get("os") == platform_os and item.get("platform", {}).get("architecture") == platform_arch), ""))')"
-          if [[ -n "$IMAGE_DIGEST" ]]; then
-            MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${IMAGE_DIGEST}")"
-          fi
-          COMPRESSED_SIZE="$(MANIFEST="$MANIFEST" "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); print(manifest["config"]["size"] + sum(layer["size"] for layer in manifest["layers"]))')"
-          LIMIT=$((750 * 1024 * 1024))
-          echo "${DIGEST} compressed_size=$((COMPRESSED_SIZE / 1024 / 1024)) MiB limit=750 MiB"
-          (( COMPRESSED_SIZE < LIMIT ))
-
-      - name: Resolve real Docker CLI test dependencies through abxpkg
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mount_env="$(
-            uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:mount_validation_binaries"
-          )"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$mount_env" >> "$GITHUB_ENV"
-
-          for binary_name in bash chmod curl exportfs mkdir mount.cifs mount.nfs smbd sudo systemctl umount; do
-            binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
-            test -L "$binary"
-            test -x "$binary"
-          done
-
-      - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          NFS_SOURCE="/srv/archivebox-ci-nfs"
-          SMB_SOURCE="/srv/archivebox-ci-smb"
-          NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
-          SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
-          mounted_nfs=0
-          mounted_smb=0
-
-          cleanup() {
-            local original_status=$? cleanup_status=0
-            trap - EXIT
-            if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
-              cleanup_status=1
-            fi
-            if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
-              cleanup_status=1
-            fi
-            if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
-              cleanup_status=1
-            fi
-            if [[ "$original_status" != "0" ]]; then
-              exit "$original_status"
-            fi
-            exit "$cleanup_status"
-          }
-          trap cleanup EXIT
-
-          "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-          "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
-          "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-          "$SUDO_BINARY" "$BASH_BINARY" -c \
-            "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
-          "$SUDO_BINARY" "$BASH_BINARY" -c \
-            "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
-
-          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
-          "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
-          "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
-          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
-          "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
-
-          "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
-          mounted_nfs=1
-          "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
-            -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
-          mounted_smb=1
-
-          NFS_TEST_DIR="$NFS_MOUNT" \
-          SMB_TEST_DIR="$SMB_MOUNT" \
-          IMAGE="${{ matrix.local_tag }}" \
-            "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
-
-      - name: Validate Docker CLI add/list and compose behavior
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        env:
-          ARCHIVEBOX_IMAGE: ${{ matrix.local_tag }}
-        run: |
-          set -Eeuo pipefail
-          "$MKDIR_BINARY" "${{ github.workspace }}/data"
-          "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
-
-          "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
-          "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
-
-          "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
-          echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add
-
-          archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
-          [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
-          [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
-
-          "$DOCKER_BINARY" compose run --rm archivebox init
-          "$DOCKER_BINARY" compose up -d --wait
-          "$DOCKER_BINARY" network connect archivebox_default www-nginx
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/public/')"
-          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js')"
-          [[ "$index_html" == *ArchiveBox* ]]
-          [[ "$admin_js" == *window.django* ]]
-
-          "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/public/')"
-          [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
-          "$DOCKER_BINARY" compose down
-
-      - name: Validate compressed image size
-        if: inputs.full_tests && !inputs.push_digests
-        env:
-          BUILD_METADATA: ${{ steps.docker_build_pr.outputs.metadata }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          LOCAL_IMAGE_ID="$("$DOCKER_BINARY" image inspect '${{ matrix.local_tag }}' --format '{{.Id}}')"
-          "$UV_BINARY" run --no-cache --no-project python - \
-            '${{ runner.temp }}/archivebox-image.tar' \
-            '${{ steps.docker_build_pr.outputs.digest }}' \
-            "$LOCAL_IMAGE_ID" \
-            "$((750 * 1024 * 1024))" <<'PY'
-          from hashlib import sha256
-          import json
-          import os
-          import sys
-          import tarfile
-
-          archive_path, pushed_digest, local_image_id, limit_text = sys.argv[1:]
-          limit = int(limit_text)
-          metadata = json.loads(os.environ["BUILD_METADATA"])
-          if metadata["containerimage.digest"] != pushed_digest:
-              raise SystemExit("Build push transaction digest does not match the action output")
-          if not pushed_digest.startswith("sha256:") or len(pushed_digest) != 71:
-              raise SystemExit(f"Invalid pushed digest: {pushed_digest}")
-
-          with tarfile.open(archive_path) as archive:
-              def read_blob(descriptor):
-                  algorithm, digest = descriptor["digest"].split(":", 1)
-                  if algorithm != "sha256":
-                      raise SystemExit(f"Unsupported OCI digest: {descriptor['digest']}")
-                  member = archive.extractfile(f"blobs/sha256/{digest}")
-                  if member is None:
-                      raise SystemExit(f"Missing OCI blob: {descriptor['digest']}")
-                  blob = member.read()
-                  if len(blob) != descriptor["size"]:
-                      raise SystemExit(f"OCI size mismatch: {descriptor['digest']}")
-                  if sha256(blob).hexdigest() != digest:
-                      raise SystemExit(f"OCI digest mismatch: {descriptor['digest']}")
-                  return blob
-
-              index_member = archive.extractfile("index.json")
-              if index_member is None:
-                  raise SystemExit("OCI export is missing index.json")
-              index = json.load(index_member)
-
-              manifests = []
-
-              def collect(descriptor):
-                  document = json.loads(read_blob(descriptor))
-                  media_type = descriptor["mediaType"]
-                  if "image.index" in media_type or "manifest.list" in media_type:
-                      for child in document["manifests"]:
-                          collect(child)
-                  elif "image.manifest" in media_type:
-                      config_type = document["config"]["mediaType"]
-                      if "image.config" in config_type or "container.image" in config_type:
-                          manifests.append(document)
-
-              for root in index["manifests"]:
-                  collect(root)
-              if len(manifests) != 1:
-                  raise SystemExit(f"Expected one runnable image manifest, found {len(manifests)}")
-
-              manifest = manifests[0]
-              config_digest = manifest["config"]["digest"]
-              if metadata["containerimage.config.digest"] != config_digest:
-                  raise SystemExit("OCI config digest does not match the pushed build result")
-              if local_image_id != config_digest:
-                  raise SystemExit("Locally tested image does not match the OCI build result")
-              descriptors = [manifest["config"], *manifest["layers"]]
-              for descriptor in descriptors:
-                  read_blob(descriptor)
-              total = sum(descriptor["size"] for descriptor in descriptors)
-
-          print(
-              f"{pushed_digest} compressed_size={total // 1024 // 1024} MiB "
-              f"limit={limit // 1024 // 1024} MiB",
-          )
-          if total > limit:
-              raise SystemExit(f"{pushed_digest} is over the compressed size limit")
-          PY
-
-      - name: Export digest
-        if: inputs.push_digests
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mkdir -p /tmp/digests
-          digest="${{ steps.docker_build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      - name: Upload digest
-        if: inputs.push_digests
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+      - name: Update README
+        uses: peter-evans/dockerhub-description@v4
        with:
-          name: ${{ matrix.artifact_name }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: archivebox/archivebox
+
+      # This ugly bit is necessary if you don't want your cache to grow forever
+      # until it hits GitHub's limit of 5GB.
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
@@ -1,158 +0,0 @@
-name: Documentation tests
-
-on:
-  workflow_call:
-
-env:
-  PYTHONIOENCODING: utf-8
-  USE_COLOR: "False"
-  SHOW_PROGRESS: "False"
-
-jobs:
-  discover:
-    name: Discover documentation examples
-    runs-on: ubuntu-24.04
-    timeout-minutes: 5
-    outputs:
-      matrix: ${{ steps.inventory.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-      - name: Install ArchiveBox development dependencies
-        run: uv sync --no-cache --locked --dev
-      - name: Resolve documentation tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
-        run: |
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          uv run --no-cache --no-sync abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries"
-          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
-          mmdc_binary="$ABXPKG_LIB_DIR/env/bin/mmdc"
-          nginx_binary="$ABXPKG_LIB_DIR/env/bin/nginx"
-          test -L "$bash_binary"
-          test -L "$mmdc_binary"
-          test -L "$nginx_binary"
-          test -x "$bash_binary"
-          test -x "$mmdc_binary"
-          test -x "$nginx_binary"
-          {
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-            echo "BASH_BINARY=$bash_binary"
-          } >> "$GITHUB_ENV"
-      - id: inventory
-        name: Validate the complete authored documentation inventory
-        run: |
-          matrix="$(uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py matrix)"
-          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
-
-  examples:
-    name: docs/${{ matrix.name }}
-    needs: discover
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 15
-    strategy:
-      fail-fast: false
-      max-parallel: 4
-      matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        with:
-          python-version: "3.13"
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-      - name: Install ArchiveBox
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        run: uv sync --no-cache --locked --dev
-      - name: Resolve documentation runtime through abxpkg
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
-        run: |
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          dependency_sources=(
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
-          )
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
-          fi
-          if [[ '${{ matrix.name }}' == ubuntu-AGENTS || '${{ matrix.name }}' == ubuntu-skills-archivebox-SKILL || '${{ matrix.name }}' == ubuntu-docs-Configuration || '${{ matrix.name }}' == ubuntu-docs-Setting-up-Authentication ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          fi
-          if [[ '${{ matrix.environment }}' == docker ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
-          elif [[ '${{ matrix.environment }}' == macos ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")
-          fi
-          uv run --no-cache --no-sync abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            "${dependency_sources[@]}"
-          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
-          python_binary="$ABXPKG_LIB_DIR/env/bin/python"
-          uv_binary="$ABXPKG_LIB_DIR/env/bin/uv"
-          test -L "$bash_binary"
-          test -x "$bash_binary"
-          test -L "$python_binary"
-          test -x "$python_binary"
-          test -L "$uv_binary"
-          test -x "$uv_binary"
-          {
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-            echo "BASH_BINARY=$bash_binary"
-            echo "PYTHON_BINARY=$python_binary"
-            echo "UV_BINARY=$uv_binary"
-          } >> "$GITHUB_ENV"
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            sudo_binary="$ABXPKG_LIB_DIR/env/bin/sudo"
-            test -L "$sudo_binary"
-            test -x "$sudo_binary"
-            echo "SUDO_BINARY=$sudo_binary" >> "$GITHUB_ENV"
-          fi
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-          fi
-          if [[ '${{ matrix.environment }}' == docker ]]; then
-            docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
-            test -L "$docker_binary"
-            test -x "$docker_binary"
-            echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV"
-          elif [[ '${{ matrix.environment }}' == macos ]]; then
-            test -L "$ABXPKG_LIB_DIR/env/bin/brew"
-            test -x "$ABXPKG_LIB_DIR/env/bin/brew"
-          fi
-      - name: Build the exact checkout's documentation image
-        if: ${{ matrix.environment == 'docker' }}
-        run: '"$DOCKER_BINARY" build --tag archivebox/archivebox:dev .'
-      - name: Run the Ubuntu, Docker, or macOS documentation lane
-        if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
-        run: uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
-      - name: Run the root documentation lane
-        if: ${{ matrix.environment == 'root' }}
-        run: |
-          "$SUDO_BINARY" \
-            --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
-            "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
@@ -0,0 +1,51 @@
+name: Build Homebrew package
+
+on:
+  workflow_dispatch:
+  push:
+
+
+jobs:
+  build:
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      # TODO: modify archivebox.rb to update src url, hashes, and dependencies
+
+      - name: Build Homebrew Bottle
+        run: |
+          pip3 install --upgrade pip setuptools wheel
+          cd brew_dist/
+          brew install --build-bottle ./archivebox.rb
+          # brew bottle archivebox
+          archivebox version
+
+      - name: Add some links to test
+        run: |
+          mkdir data && cd data
+          archivebox init --setup
+          archivebox add 'https://example.com'
+          archivebox version
+          archivebox status
+
+      # - name: Commit built package
+      #   run: |
+      #     cd brew_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Homebrew package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/homebrew-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: brew_dist
+
+      # TODO: push bottle homebrew core PR with latest changes
@@ -0,0 +1,58 @@
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Build GitHub Pages website
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["dev"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Copy README.md into place
+        run: |
+          rm ./website/README.md
+          cp ./README.md ./website/README.md
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2
@@ -1,38 +1,35 @@
 name: Run linters

 on:
-  workflow_call:
+  workflow_dispatch:
+  push:

 env:
-  UV_NO_SOURCES: "1"
+  MAX_LINE_LENGTH: 110

 jobs:
  lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v2
        with:
          submodules: true
          fetch-depth: 1

      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/setup-python@v1
        with:
-          python-version: "3.13"
+          python-version: 3.9
          architecture: x64

-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+      - name: Install flake8
+        run: |
+          pip install flake8

-      - name: Verify checked dependency lock
-        run: uv lock --no-cache --check
-
-      - name: Install dependencies with uv
-        run: uv sync --no-cache --locked --dev --extra sonic --extra debug
-
-      - name: Run prek
-        run: uv run --no-cache --no-sync --no-sources prek run --all-files
+      - name: Lint with flake8
+        run: |
+          cd archivebox
+          # one pass for show-stopper syntax errors or undefined names
+          flake8 . --count --show-source --statistics
+          # one pass for small stylistic things
+          flake8 . --count --max-line-length="$MAX_LINE_LENGTH" --statistics
@@ -1,243 +1,78 @@
 name: Build Pip package

 on:
-  workflow_call:
-    inputs:
-      full_tests:
-        description: Run the full package compatibility matrix instead of one release smoke check
-        required: false
-        default: true
-        type: boolean
+  workflow_dispatch:
+  push:

-env:
-  UV_VERSION: "0.11.3"

 jobs:
  build:
-    name: build distributions
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+    runs-on: ubuntu-22.04

-      - name: Set up Python 3.13
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-          architecture: x64
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: ${{ env.UV_VERSION }}
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Build every distribution
-        run: |
-          set -Eeuo pipefail
-          uv build --no-cache --all-packages --no-sources --out-dir dist --clear
-
-          shopt -s nullglob
-          artifacts=(dist/*)
-          wheels=(dist/archivebox-*.whl)
-          sdists=(dist/archivebox-*.tar.gz)
-          [[ "${#artifacts[@]}" -gt 0 ]]
-          [[ "${#wheels[@]}" -eq 1 ]]
-          [[ "${#sdists[@]}" -eq 1 ]]
-          printf '%s\n' "$GITHUB_SHA" > dist/COMMIT_SHA
-          uv run --no-cache --no-project python - <<'PY'
-          import hashlib
-          from pathlib import Path
-
-          root = Path("dist")
-          wheels = list(root.glob("archivebox-*.whl"))
-          sdists = list(root.glob("archivebox-*.tar.gz"))
-          if len(wheels) != 1 or len(sdists) != 1:
-              raise SystemExit("Expected exactly one ArchiveBox wheel and one sdist")
-          artifacts = [root / "COMMIT_SHA", wheels[0], sdists[0]]
-          checksum_lines = []
-          for artifact in artifacts:
-              digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
-              line = f"{digest}  {artifact.name}"
-              checksum_lines.append(line)
-              print(line)
-          Path("dist/SHA256SUMS").write_text("\n".join(checksum_lines) + "\n")
-          PY
-
-      - name: Release wheel import and CLI smoke
-        if: ${{ !inputs.full_tests }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          shopt -s nullglob
-          wheels=(dist/archivebox-*.whl)
-          [[ "${#wheels[@]}" -eq 1 ]]
-          uv sync --locked --no-cache --no-sources --no-dev --no-install-project
-          uv pip install --no-cache --no-deps --python .venv/bin/python "${wheels[0]}"
-          unset PYTHONPATH
-          cd "$(mktemp -d)"
-          VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project python -c 'import archivebox'
-          VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project archivebox version
-
-      - name: Upload every distribution
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: python-distributions
-          path: dist/*
-          if-no-files-found: error
-          retention-days: 7
-
-  install-smoke:
-    name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }}
-    if: inputs.full_tests
-    needs: build
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-24.04, macos-15]
-        python: ["3.13", "3.14"]
-        artifact: [wheel, sdist]
+    permissions:
+      id-token: write

    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
        with:
+          submodules: true
          fetch-depth: 1

-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+      - name: Set up Python
+        uses: actions/setup-python@v5
        with:
-          python-version: ${{ matrix.python }}
+          python-version: '3.11'
+          architecture: x64

-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
-          version: ${{ env.UV_VERSION }}
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0

-      - name: Download distributions
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+      - uses: pdm-project/setup-pdm@v3
        with:
-          name: python-distributions
-          path: ${{ runner.temp }}/python-distributions
+          python-version: '3.11'
+          cache: true

-      - name: Install and smoke-test the built ${{ matrix.artifact }}
-        shell: bash
+      - name: Install dependencies
+        run: pdm install --fail-fast --no-lock --dev --group=':all' --no-self
+
+      - name: Build package
        run: |
-          set -Eeuo pipefail
-          shopt -s nullglob
+          pdm build

-          case "${{ matrix.artifact }}" in
-            wheel) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.whl) ;;
-            sdist) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.tar.gz) ;;
-            *) exit 2 ;;
-          esac
-          [[ "${#candidates[@]}" -eq 1 ]]
-          artifact="${candidates[0]}"
+      - name: Install from build
+        run: pip install ./dist/archivebox-*.whl

-          smoke_root="$(mktemp -d "$RUNNER_TEMP/archivebox-package-smoke.XXXXXX")"
-          trap 'rm -rf "$smoke_root"' EXIT
-          smoke_env="$smoke_root/env"
-          data_dir="$smoke_root/data"
-          mkdir -p "$data_dir"
+      - name: Add some links to test
+        run: |
+          mkdir data && cd data
+          archivebox init
+          archivebox add 'https://example.com'
+          archivebox version
+          archivebox status

-          dependency_artifacts=()
-          while IFS= read -r dependency_artifact; do
-            dependency_artifacts+=("$dependency_artifact")
-          done < <(uv run --no-cache --no-project python - <<'PY'
-          import tomllib
+      #- name: Publish package distributions to PyPI
+      #  run: pdm publish --no-build

-          locked = {
-              package["name"]: package
-              for package in tomllib.load(open("uv.lock", "rb"))["package"]
-          }
-          for package_name in ("abx-dl", "abx-plugins", "abxpkg", "abxbus"):
-              package = locked[package_name]
-              wheels = package.get("wheels") or []
-              if len(wheels) != 1:
-                  raise SystemExit(f"Expected exactly one locked wheel for {package_name}")
-              print(wheels[0]["url"])
-          PY
-          )
+      # - name: Push build to PyPI
+      #   run: |
+      #     cd pip_dist/
+      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
+      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}

-          uv venv --python "${{ matrix.python }}" "$smoke_env"
-          uv pip install --no-cache \
-            --python "$smoke_env" \
-            "${dependency_artifacts[@]}" \
-            "$artifact"
+      # - name: Commit built package
+      #   run: |
+      #     cd pip_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Pip package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/pip-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: pip_dist

-          unset PYTHONPATH
-          cd "$data_dir"
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync python - <<'PY'
-          import os
-          from pathlib import Path
-
-          import archivebox
-
-          installed_path = Path(archivebox.__file__).resolve()
-          environment_path = Path(os.environ["VIRTUAL_ENV"]).resolve()
-          workspace_path = Path(os.environ["GITHUB_WORKSPACE"]).resolve()
-          assert installed_path.is_relative_to(environment_path), (installed_path, environment_path)
-          assert not installed_path.is_relative_to(workspace_path), (installed_path, workspace_path)
-          print(installed_path)
-          PY
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox version
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox init
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox status
-
-          if [[ "${{ runner.os }}" == "Linux" ]]; then
-            root_tool_dir="/root/archivebox-package-smoke-${{ matrix.artifact }}-${{ matrix.python }}"
-            uv_binary="$(command -v uv)"
-            sudo rm -rf "$root_tool_dir"
-            sudo mkdir -p "$root_tool_dir/data"
-
-            root_install_args=()
-            for dependency_artifact in "${dependency_artifacts[@]}"; do
-              root_install_args+=(--with "$dependency_artifact")
-            done
-            sudo UV_TOOL_DIR="$root_tool_dir/tools" UV_TOOL_BIN_DIR="$root_tool_dir/bin" \
-              "$uv_binary" tool install --no-cache --python "${{ matrix.python }}" \
-              "${root_install_args[@]}" "$artifact"
-
-            sudo useradd --system --create-home --home-dir /var/lib/archivebox --shell /bin/bash archivebox 2>/dev/null || true
-            sudo rm -rf /var/lib/archivebox
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data' && archivebox init"
-            sudo stat -c %U /var/lib/archivebox | grep -qx archivebox
-            sudo stat -c %U "$root_tool_dir/data" | grep -qx archivebox
-            if sudo find "$root_tool_dir/data" -xdev -user root -print -quit | grep -q .; then
-              echo "Root-owned files remain in $root_tool_dir/data" >&2
-              exit 1
-            fi
-
-            sudo mkdir "$root_tool_dir/external"
-            sudo ln -s "$root_tool_dir/external" "$root_tool_dir/data/external-link"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data' && archivebox status"
-            sudo stat -c %U "$root_tool_dir/external" | grep -qx root
-
-            sudo mkdir "$root_tool_dir/data-universal-init"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-universal-init' && archivebox status --init"
-            sudo stat -c %U "$root_tool_dir/data-universal-init" | grep -qx archivebox
-
-            sudo mkdir "$root_tool_dir/data-install"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-install' && archivebox install --dry-run"
-            sudo stat -c %U "$root_tool_dir/data-install" | grep -qx archivebox
-
-            sudo useradd --create-home archivebox-smoke-user
-            smoke_user_group="$(id -gn archivebox-smoke-user)"
-            sudo mkdir "$root_tool_dir/data-mixed-owner"
-            sudo chown archivebox-smoke-user:root "$root_tool_dir/data-mixed-owner"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-mixed-owner' && archivebox init"
-            sudo stat -c %U:%G "$root_tool_dir/data-mixed-owner" | grep -qx "archivebox-smoke-user:$smoke_user_group"
-            if sudo find "$root_tool_dir/data-mixed-owner" -xdev -user root -print -quit | grep -q .; then
-              echo "Root-owned files remain in $root_tool_dir/data-mixed-owner" >&2
-              exit 1
-            fi
-            sudo rm -rf "$root_tool_dir"
-          fi
@@ -1,51 +0,0 @@
-name: Release candidate
-
-on:
-  push:
-    branches: [dev]
-
-permissions:
-  actions: read
-  contents: write
-  packages: write
-
-concurrency:
-  group: release-candidate-dev
-  cancel-in-progress: true
-
-jobs:
-  prepare:
-    runs-on: ubuntu-24.04
-    outputs:
-      action: ${{ steps.version.outputs.action }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          token: ${{ secrets.RELEASE_GH_TOKEN }}
-
-      - id: version
-        uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
-        with:
-          package: archivebox
-          branch: dev
-          tag-prefix: v
-          version-scheme: rc
-
-  python-artifacts:
-    needs: prepare
-    if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved'
-    uses: ./.github/workflows/pip.yml
-    with:
-      full_tests: false
-    secrets: inherit
-
-  docker-digests:
-    needs: prepare
-    if: needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved'
-    uses: ./.github/workflows/docker.yml
-    with:
-      push_digests: true
-      full_tests: false
-    secrets: inherit
@@ -1,331 +0,0 @@
-name: Release
-
-on:
-  workflow_run:
-    workflows: [CI, Release candidate]
-    types: [completed]
-    branches: [dev, main]
-
-permissions: {}
-
-concurrency:
-  group: release-${{ github.repository }}-${{ github.event.workflow_run.head_branch }}
-  cancel-in-progress: false
-
-env:
-  DOCKERHUB_IMAGE: archivebox/archivebox
-  GHCR_IMAGE: ghcr.io/archivebox/archivebox
-  RELEASE_BRANCH: ${{ github.event.workflow_run.head_branch }}
-  RELEASE_SHA: ${{ github.event.workflow_run.head_sha }}
-  CI_RUN_ID: ${{ github.event.workflow_run.id }}
-
-jobs:
-  candidate:
-    if: >-
-      github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.event == 'push' &&
-      ((github.event.workflow_run.head_branch == 'dev' && github.event.workflow_run.name == 'Release candidate') ||
-       (github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.name == 'CI'))
-    runs-on: ubuntu-24.04
-    outputs:
-      ready: ${{ steps.owner.outputs.ready }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          ref: ${{ env.RELEASE_SHA }}
-      - id: owner
-        shell: bash
-        run: |
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            echo 'ready=true' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
-          TARGET="$(git ls-remote origin "refs/tags/release-candidate/$VERSION" | awk '{print $1}')"
-          [[ "$TARGET" == "$RELEASE_SHA" ]] && echo 'ready=true' >> "$GITHUB_OUTPUT" || echo 'ready=false' >> "$GITHUB_OUTPUT"
-
-  python-release:
-    name: Publish tested Python artifacts
-    needs: candidate
-    if: needs.candidate.outputs.ready == 'true'
-    runs-on: ubuntu-24.04
-    environment: pypi
-    permissions:
-      actions: read
-      contents: write
-      id-token: write
-    outputs:
-      version: ${{ steps.version.outputs.version }}
-      tag: ${{ steps.version.outputs.tag }}
-      release_ready: ${{ steps.release-state.outputs.ready }}
-    steps:
-      - name: Checkout the tested commit
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          ref: ${{ env.RELEASE_SHA }}
-          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      - name: Resolve release binaries through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" >/dev/null
-          for name in uv gh git jq curl; do
-            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
-            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
-          done
-          {
-            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
-            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
-            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-            echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
-          } >> "$GITHUB_ENV"
-
-      - name: Download exact tested distributions
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: python-distributions
-          path: ${{ runner.temp }}/python-distributions
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.CI_RUN_ID }}
-
-      - name: Record release version
-        id: version
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$($UV_BINARY run --no-cache --no-project python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
-          echo "tag=v$VERSION" >> "$GITHUB_OUTPUT"
-
-      - name: Validate release branch and version
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION='${{ steps.version.outputs.version }}'
-          case "$RELEASE_BRANCH" in
-            main)
-              ;;
-            dev)
-              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]] || {
-                echo "Refusing to publish ArchiveBox $VERSION from dev; only archivebox:main may publish non-prerelease versions." >&2
-                exit 1
-              }
-              ;;
-            *)
-              echo "Refusing to publish ArchiveBox $VERSION from unsupported branch $RELEASE_BRANCH; only archivebox:dev and archivebox:main may publish." >&2
-              exit 1
-              ;;
-          esac
-
-      - name: Publish the exact tested distributions
-        env:
-          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-          RELEASE_DISTRIBUTIONS_DIR: ${{ runner.temp }}/python-distributions
-        run: ./bin/release.sh
-
-      - name: Confirm this tested commit owns the release tag
-        id: release-state
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          TARGET=""
-          while read -r sha _; do
-            TARGET="$sha"
-          done < <("$GIT_BINARY" ls-remote origin "refs/tags/${{ steps.version.outputs.tag }}")
-          if [[ "$TARGET" == "$RELEASE_SHA" ]]; then
-            echo "ready=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "ready=false" >> "$GITHUB_OUTPUT"
-          fi
-
-  docker-release:
-    name: Publish tested multiarch image
-    if: >-
-      github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.event == 'push'
-    needs: python-release
-    runs-on: ubuntu-24.04
-    permissions:
-      actions: read
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout the tested commit
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-          ref: ${{ env.RELEASE_SHA }}
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      - name: Resolve release and Docker binaries through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
-          for name in uv git jq docker; do
-            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
-            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
-          done
-          {
-            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
-            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-            echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
-          } >> "$GITHUB_ENV"
-
-      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
-        with:
-          install: true
-
-      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ github.token }}
-
-      - name: Download exact tested image digests
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          path: /tmp/digests
-          pattern: digest-*
-          merge-multiple: true
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.CI_RUN_ID }}
-
-      - name: Verify digest set and collect tags
-        id: docker_meta
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
-          [[ "${#DIGESTS[@]}" -eq 2 ]]
-          for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
-
-          VERSION='${{ needs.python-release.outputs.version }}'
-          SHORT_SHA="${RELEASE_SHA::12}"
-          TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}^{}")"
-          TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
-          if [[ -z "$TAG_TARGET" ]]; then
-            TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
-            TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
-          fi
-          {
-            echo 'dockerhub_tags<<EOF'
-            if [[ "$RELEASE_BRANCH" == dev ]]; then
-              echo "${DOCKERHUB_IMAGE}:dev"
-            elif [[ "$RELEASE_BRANCH" == main ]]; then
-              echo "${DOCKERHUB_IMAGE}:latest"
-            fi
-            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
-            echo 'EOF'
-            echo 'ghcr_tags<<EOF'
-            if [[ "$RELEASE_BRANCH" == dev ]]; then
-              echo "${GHCR_IMAGE}:dev"
-            elif [[ "$RELEASE_BRANCH" == main ]]; then
-              echo "${GHCR_IMAGE}:latest"
-            fi
-            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
-            echo 'EOF'
-          } >> "$GITHUB_OUTPUT"
-
-      - name: Create Docker Hub manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
-          TAG_ARGS=()
-          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
-          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Create GHCR manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
-          TAG_ARGS=()
-          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
-          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Verify published Docker images run
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION='${{ needs.python-release.outputs.version }}'
-          SHORT_SHA="${RELEASE_SHA::12}"
-          for image in "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"; do
-            "$DOCKER_BINARY" buildx imagetools inspect "$image" >/dev/null
-            INSTALLED_VERSION="$("$DOCKER_BINARY" run --rm "$image" --version)"
-            test "$INSTALLED_VERSION" = "$VERSION"
-          done
-
-      - name: Update Docker Hub README
-        uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: archivebox/archivebox
-
-  cascade:
-    name: Cascade verified release
-    if: needs.python-release.outputs.release_ready == 'true'
-    needs:
-      - python-release
-      - docker-release
-    permissions:
-      actions: read
-      contents: read
-    uses: ArchiveBox/monorepo/.github/workflows/cascade-release.yml@main
-    with:
-      upstream_package: archivebox
-      upstream_version: ${{ needs.python-release.outputs.version }}
-      upstream_artifact_name: python-distributions
-    secrets:
-      release_token: ${{ secrets.RELEASE_GH_TOKEN }}
@@ -1,154 +0,0 @@
-name: Parallel Tests
-
-on:
-  workflow_call:
-
-env:
-  PYTHONIOENCODING: utf-8
-  PYTHONLEGACYWINDOWSSTDIO: utf-8
-  USE_COLOR: False
-  UV_NO_SOURCES: "1"
-  CI_PYTHON_VERSION: "3.13"
-
-jobs:
-  discover-tests:
-    name: Discover every test
-    runs-on: ubuntu-24.04
-    outputs:
-      tests: ${{ steps.discover.outputs.tests }}
-
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: ${{ env.CI_PYTHON_VERSION }}
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Resolve discovery tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          test -n "$ABXPKG_SPEC"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PYTHON_BINARY="$pythonLocation/bin/python"
-          test -x "$PYTHON_BINARY"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
-          {
-            echo "PYTHON_BINARY=$ABXPKG_LIB_DIR/env/bin/python"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-          } >> "$GITHUB_ENV"
-          test -L "$ABXPKG_LIB_DIR/env/bin/python"
-          test -x "$ABXPKG_LIB_DIR/env/bin/python"
-          test -L "$ABXPKG_LIB_DIR/env/bin/jq"
-          test -x "$ABXPKG_LIB_DIR/env/bin/jq"
-
-      - name: Discover every test exactly once
-        id: discover
-        run: |
-          set -Eeuo pipefail
-          mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
-          test "${#output[@]}" -eq 2
-          echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
-          echo "${output[0]}"
-          "$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"
-
-  tests:
-    name: ${{ matrix.test.name }}
-    needs: discover-tests
-    runs-on: ubuntu-24.04
-    env:
-      ABXPKG_LIB_DIR: /tmp/archivebox-lib
-      CHROME_HEADLESS: "true"
-      DATA_DIR: /tmp/archivebox-test-data
-      PERSONAS_DIR: /tmp/abx-personas
-      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
-
-    strategy:
-      fail-fast: false
-      matrix:
-        test: ${{ fromJson(needs.discover-tests.outputs.tests) }}
-
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: ${{ env.CI_PYTHON_VERSION }}
-          architecture: x64
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Resolve optional build dependencies through abxpkg
-        if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
-        env:
-          TEST_EXTRA: ${{ matrix.test.extra }}
-          TEST_PATHS: ${{ matrix.test.paths_arg }}
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          test -n "$ABXPKG_SPEC"
-          dependency_sources=()
-          if [[ -n "$TEST_EXTRA" ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          fi
-          if [[ "$TEST_PATHS" == *test_postgres_backend* ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:postgres_binaries")
-          fi
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-            --install \
-            --no-cache \
-            --json \
-            --lib="$ABXPKG_LIB_DIR" \
-            "${dependency_sources[@]}"
-
-      - name: Install test dependencies
-        env:
-          TEST_EXTRA: ${{ matrix.test.extra }}
-        run: |
-          set -Eeuo pipefail
-          extra_args=()
-          if [[ -n "$TEST_EXTRA" ]]; then
-            extra_args=(--extra "$TEST_EXTRA")
-          fi
-          uv sync --no-cache --locked --dev "${extra_args[@]}"
-
-      - name: Run ${{ matrix.test.name }}
-        env:
-          TEST_PATHS: ${{ matrix.test.paths_arg }}
-        run: |
-          set -Eeuo pipefail
-          read -r -a test_paths <<< "$TEST_PATHS"
-          for test_path in "${test_paths[@]}"; do
-            test -f "$test_path"
-          done
-          for test_path in "${test_paths[@]}"; do
-            test_name="$(basename "$test_path" .py)"
-            test_slug="${test_name//[^A-Za-z0-9_.-]/_}"
-            export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
-            export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
-            export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
-            mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
-            uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" \
-              --basetemp="$RUNNER_TEMP/pytest/$test_slug"
-          done
@@ -1,63 +1,158 @@
-name: Install and CLI Compatibility
-
-on:
-  workflow_call:
+name: Run tests
+on: [push]

 env:
  DOCKER_IMAGE: archivebox-ci
-  ARCHIVEBOX_IMAGE: archivebox-ci
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False
-  UV_NO_SOURCES: "1"

 jobs:
-  install_cli_platform:
-    name: CLI install / ${{ matrix.os_name }} / Python ${{ matrix.python }}
+  python_tests:
    runs-on: ${{ matrix.os }}

    strategy:
-      fail-fast: false
      matrix:
-        include:
-          - os: ubuntu-24.04
-            os_name: linux
-            python: "3.14"
-          - os: macos-15
-            os_name: macOS
-            python: "3.13"
-          - os: macos-15
-            os_name: macOS
-            python: "3.14"
+        os: [ubuntu-22.04]
+        # os: [ubuntu-22.04, macos-latest, windows-latest]
+        python: [3.11]

    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1

-      ### Set up Python
+      ### Setup Python & JS Languages
      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
+          architecture: x64

-      - name: Install uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          node-version: 20.10.0

-      - name: Install dependencies with uv
-        run: uv sync --no-cache --locked --dev
+      - name: Setup PDM
+        uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true

-      - name: ArchiveBox full install check
+      ### Install Python & JS Dependencies
+      - name: Get pip cache dir
+        id: pip-cache
        run: |
-          set -Eeuo pipefail
-          DATA_DIR="$PWD/tests/out/data"
-          mkdir -p "$DATA_DIR"
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox init
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox install
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox version
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox status
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: Cache pip
+        uses: actions/cache@v3
+        id: cache-pip
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python }}-venv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
+
+      - name: Install pip dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel pytest bottle build
+          python -m pip install -r requirements.txt
+          python -m pip install -e .[sonic,ldap]
+
+      - name: Get npm cache dir
+        id: npm-cache
+        run: |
+          echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
+
+      - name: Cache npm
+        uses: actions/cache@v3
+        id: cache-npm
+        with:
+          path: ${{ steps.npm-cache.outputs.dir }}
+          key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node_modules
+
+      - name: Install npm requirements
+        run: |
+          npm install
+          echo "SINGLEFILE_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/single-file" >> $GITHUB_ENV
+          echo "READABILITY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor" >> $GITHUB_ENV
+          echo "MERCURY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/mercury-parser" >> $GITHUB_ENV
+
+      ### Run the tests
+      - name: Directory listing for debugging
+        run: |
+          pwd
+          ls
+
+      - name: Archivebox version
+        run: |
+          archivebox version
+
+      - name: Test built package with pytest
+        # TODO: remove this exception for windows once we get tests passing on that platform
+        if: ${{ !contains(matrix.os, 'windows') }}
+        run: |
+          python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist
+
+  docker_tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
+      # - uses: satackey/action-docker-layer-caching@v0.0.8
+
+      - name: Build image
+        run: |
+          docker build . -t "$DOCKER_IMAGE"
+
+      - name: Init data dir
+        run: |
+          mkdir "${{ github.workspace }}/data"
+          docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init
+
+      - name: Run test server
+        run: |
+          sudo bash -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
+          docker run --name www-nginx -p 80:80 -d nginx
+
+      - name: Add link
+        run: |
+          docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local
+
+      - name: Add stdin link
+        run: |
+          echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add
+
+      - name: List links
+        run: |
+          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
+          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }
+
+      - name: Start docker-compose stack
+        run: |
+          docker-compose run archivebox init
+          docker-compose up -d
+          sleep 5
+          curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
+          curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'
+
+      - name: Check added urls show up in index
+        run: |
+          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
+          curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
+          docker-compose down || true
@@ -1,36 +1,22 @@
 .DS_Store
 ._*
-/.heartbeat.json
-/package.json
-/package-lock.json
 *.pyc
 __pycache__/
 .mypy_cache/
 .eggs/
 tests/out/
-/docs/screenshots/
-/publicsite/screenshots/
-
-# Coverage
-.coverage
-.coverage.*
-coverage.json
-coverage/
-htmlcov/

 # Python and Node dependencies
 venv/
 .venv/
 .docker-venv/
 node_modules/
-typings/

 # Ignore dev lockfiles (should always be built fresh)
 pdm.dev.lock
 requirements-dev.txt

 # Packaging artifacts
-requirements.txt
 .pdm-python
 .pdm-build
 archivebox.egg-info
@@ -39,32 +25,13 @@ build/
 dist/

 # Data folders
-lib/
-out/
-tmp/
 data/
 data*/
-archivebox/tests/data/
-archive/
 output/
-logs/
-/-
-/personas/
-/sources/
 index.sqlite3
-queue.sqlite3
 *.sqlite*
 data.*
-.archivebox_id
-ArchiveBox.conf
-*.stdout
-*.stderr
-*.log
-.tmp/

 # vim
 *.sw?
 .vscode
-
-# Local Claude Code task locks (not project state)
-.claude/scheduled_tasks.lock
@@ -0,0 +1,31 @@
+[submodule "docs"]
+    path = docs
+    url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
+
+[submodule "deb_dist"]
+    path = deb_dist
+    url = https://github.com/ArchiveBox/debian-archivebox.git
+[submodule "brew_dist"]
+	path = brew_dist
+	url = https://github.com/ArchiveBox/homebrew-archivebox.git
+[submodule "pip_dist"]
+	path = pip_dist
+	url = https://github.com/ArchiveBox/pip-archivebox.git
+[submodule "docker"]
+	path = docker
+	url = https://github.com/ArchiveBox/docker-archivebox.git
+[submodule "archivebox/vendor/base32-crockford"]
+	path = archivebox/vendor/base32-crockford
+	url = https://github.com/jbittel/base32-crockford
+[submodule "archivebox/vendor/pocket"]
+	path = archivebox/vendor/pocket
+	url = https://github.com/tapanpandita/pocket
+[submodule "archivebox/vendor/django-taggit"]
+	path = archivebox/vendor/django-taggit
+	url = https://github.com/jazzband/django-taggit
+[submodule "archivebox/vendor/python-atomicwrites"]
+	path = archivebox/vendor/python-atomicwrites
+	url = https://github.com/untitaker/python-atomicwrites
+[submodule "archivebox/vendor/pydantic-pkgr"]
+	path = archivebox/vendor/pydantic-pkgr
+	url = https://github.com/ArchiveBox/pydantic-pkgr
@@ -0,0 +1,19 @@
+tests/
+archivebox/
+archivebox.egg-info/
+build/
+dist/
+docs/
+etc/
+.github
+.gitmodules
+.dockerignore
+.flake8
+CNAME
+_config.yml
+docker-compose.yaml
+docker-compose.yml
+Dockerfile
+MANIFEST.in
+Pipfile
+setup.py
@@ -1,67 +0,0 @@
-default_language_version:
-  python: python3.13
-
-repos:
-  - repo: https://github.com/asottile/yesqa
-    rev: v1.5.0
-    hooks:
-      - id: yesqa
-
-  - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.1
-    hooks:
-      - id: codespell
-        additional_dependencies:
-          - tomli
-
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.20.0
-    hooks:
-      - id: pyupgrade
-        args: [--py313-plus]
-
-  - repo: https://github.com/asottile/add-trailing-comma
-    rev: v3.1.0
-    hooks:
-      - id: add-trailing-comma
-
-  - repo: local
-    hooks:
-      - id: ruff-format
-        name: ruff-format
-        entry: uv run --active --no-sync --no-sources ruff format
-        language: system
-        types_or: [python, pyi]
-      - id: ruff-check
-        name: ruff-check
-        entry: uv run --active --no-sync --no-sources ruff check --fix
-        language: system
-        types_or: [python, pyi]
-
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
-    hooks:
-      - id: check-ast
-      - id: check-toml
-      - id: check-yaml
-        exclude: ^\.github/workflows/homebrew\.yml$
-      - id: check-json
-      - id: check-merge-conflict
-      - id: check-symlinks
-      - id: destroyed-symlinks
-      - id: check-case-conflict
-      - id: check-illegal-windows-names
-      - id: check-shebang-scripts-are-executable
-        exclude: ^(archivebox/.*\.py|archivebox/tests/.*\.py|archivebox/personas/export_browser_state\.js)$
-      - id: mixed-line-ending
-      - id: fix-byte-order-marker
-      - id: end-of-file-fixer
-      - id: detect-private-key
-      - id: debug-statements
-      - id: forbid-submodules
-        exclude: ^docs$
-      - id: check-added-large-files
-        args: ["--maxkb=600"]
-      - id: name-tests-test
-        args: ["--pytest-test-first"]
-        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$|import_helpers\.py$)
@@ -0,0 +1,33 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+submodules:
+  include: all
+  recursive: true
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    nodejs: "20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+  - pdf
+  - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt
@@ -1,87 +0,0 @@
-# ArchiveBox Agent Guide
-
-ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
-
-## Shared Standards
-
- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
- Start behavior fixes with a red failing test when a test is requested or practical.
- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
-
-## Concurrency Contract
-
- A collection has one orchestrator at a time. Local PID/process checks may warn about obvious same-machine duplicates, but must not claim to enforce ownership across machines or shared filesystems.
- SQLite remains supported with concurrent short writes from CLI and server processes. Keep the SQLite database on a local filesystem, never NFS/SMB, and use short autocommit/CAS updates instead of long transactions or database locks.
- Network calls, hook execution, filesystem migrations, and other long work belong in the orchestrator. Never hold a database transaction or lock across that work.
- PostgreSQL and shared data directories are the path to future multi-machine scheduling. Coordinate that work through database-backed per-crawl/per-snapshot claims; do not introduce file leases or timer-based orchestrator election.
-
-## Development Setup
-
-```bash
-uv sync --dev --all-extras
-mkdir -p data
-cd data
-uv run --project .. archivebox init --install
-```
-
-Run collection commands from inside an initialized data directory:
-
-```bash
-cd data
-uv run --project .. archivebox status
-uv run --project .. archivebox add --plugins=parse_txt_urls 'https://example.com/'
-uv run --project .. archivebox run
-```
-
-## User-Facing Setup
-
-Recommended CLI install:
-
-```bash
-uv tool install --python 3.13 --prerelease explicit --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
-mkdir -p ~/archivebox/data
-cd ~/archivebox/data
-archivebox init --install
-archivebox add --plugins=parse_txt_urls 'https://example.com/'
-```
-
-Alternative install methods:
-
- Docker Compose / Docker
- Homebrew
- Debian package
- pip
-
-## Basic Usage
-
-```bash
-cd ~/archivebox/data
-archivebox version
-archivebox help
-archivebox status
-archivebox install
-archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
-archivebox list --json --with-headers
-archivebox search 'example'
-archivebox update --filter-type=domain example.com
-archivebox remove --yes --delete --filter-type=exact 'https://example.com/docs-basic-usage'
-archivebox run
-```
-
-## Verification
-
-Use targeted tests for focused work:
-
-```bash
-uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
-uv run prek run --all-files
-```
-
-Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.
@@ -1,212 +1,22 @@
-# syntax=docker/dockerfile:1.7
+# This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
+#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file
+# Usage:
+#     git submodule update --init --recursive
+#     git pull --recurse-submodules
+#     docker build . -t archivebox --no-cache
+#     docker run -v "$PWD/data":/data archivebox init
+#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
+#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
+#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
+# Multi-arch build:
+#     docker buildx create --use
+#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
+#
+# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).

-# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns sonic, supervisor, Django, and the app runtime.
-# Build abx-dl first, then point this file at it:
-#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
-#       --build-context abxbus=../abxbus \
-#       --build-context abxpkg=../abxpkg \
-#       --build-context abx-plugins=../abx-plugins \
-#       -t archivebox/abx-dl:dev
-#   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.274 \
-#       -t archivebox:multistage

-ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.274
-
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
-
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-ARG ARCHIVEBOX_COMMIT_HASH=""
-
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_NO_COMPILE=1 \
-    PIP_ONLY_BINARY=aiohttp \
-    npm_config_loglevel=error
-
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=24
-
-ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_ARCHIVEBOX_UID=911 \
-    DEFAULT_ARCHIVEBOX_GID=911 \
-    IN_DOCKER=True
-
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    CONFIG_DIR=/opt/archivebox \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
-    PERSONAS_DIR=/data/personas \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/venv/bin/python3 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-ENV HOME=/home/archivebox \
-    XDG_CONFIG_HOME=/home/archivebox/.config \
-    XDG_CACHE_HOME=/opt/archivebox/lib/cache \
-    TIMEOUT=600
-
-ENV UV_COMPILE_BYTECODE=false \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv \
-    VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:$PATH"
-
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
-WORKDIR "$CODE_DIR"
-
-RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
-    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env node \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env uv \
-    ) | tee -a /VERSION.txt
-
-FROM archivebox-runtime-base AS archivebox-builder
-
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    <<'EOF'
-echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
-echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache
-echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends
-echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests
-rm -f /etc/apt/apt.conf.d/docker-clean
-apt-get update -qq
-apt-get install -qq -y --no-install-recommends \
-    build-essential gcc libldap2-dev libsasl2-dev libssl-dev x11-utils
-# Extend abx-dl's existing venv instead of clearing it. Clearing and later
-# copying a complete replacement over the base duplicates every venv byte in
-# the final overlay history even when most packages are unchanged.
-/usr/bin/uv pip install --no-cache setuptools pip wheel
-
-mkdir -p /tmp/archivebox-uv-project
-/venv/bin/python - <<'PY'
-from pathlib import Path
-import json
-import re
-import urllib.request
-
-source = Path("/app/pyproject.toml")
-target = Path("/tmp/archivebox-uv-project/pyproject.toml")
-text = source.read_text()
-text = text.replace(
-    'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
-    'environments = ["sys_platform == \'linux\'"]',
-)
-
-# Docker builds need the just-published internal abx wheels immediately, but
-# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
-# Docker-only dependency view from the version JSON so the published package
-# metadata stays normal while image builds remain resumable after a release.
-for package in ("abxbus", "abxpkg", "abx-plugins", "abx-dl"):
-    match = re.search(
-        rf'"{re.escape(package)}(?P<operator>==|>=)(?P<version>[^"]+)"',
-        text,
-    )
-    if not match:
-        continue
-    version = match.group("version")
-    with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
-        data = json.load(response)
-    wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
-    text = re.sub(
-        rf'"{re.escape(package)}(?:==|>=)[^"]+"',
-        f'"{package} @ {wheel_url}"',
-        text,
-        count=1,
-    )
-
-target.write_text(text)
-PY
-
-/usr/bin/uv sync \
-    --project /tmp/archivebox-uv-project \
-    --no-cache \
-    --no-dev \
-    --inexact \
-    --no-install-project \
-    --no-install-workspace \
-    --no-sources
-builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
-ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
-/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
-while IFS= read -r -d '' native_library; do
-    magic=''
-    if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
-        "$builder_abxpkg_lib_dir/env/bin/strip" --strip-unneeded "$native_library" || exit $?
-    fi
-done < /tmp/archivebox-native-libraries
-rm -f /tmp/archivebox-native-libraries
-rm -f /venv/bin/uv /venv/bin/uvx
-abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils build-essential gcc libldap2-dev libsasl2-dev libssl-dev
-abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get autoremove -y
-rm -rf "$builder_abxpkg_lib_dir"
-rm -rf /venv/lib/python3.*/site-packages/pip* \
-    /venv/lib/python3.*/site-packages/wheel* \
-    /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
-echo 'Binary::apt::APT::Keep-Downloaded-Packages "0";' > /etc/apt/apt.conf.d/99keep-cache
-rm -rf /var/lib/apt/lists/* /tmp/archivebox-uv-project
-EOF
-
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-# Compile only ArchiveBox's installed package. Touching all of /venv here would
-# copy every inherited abx-dl file into a new layer merely to change its mtime.
-RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && COMMIT_HASH="$( \
-        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-            echo "$ARCHIVEBOX_COMMIT_HASH"; \
-        elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
-            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
-            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-                echo "$HEAD_REF"; \
-            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
-                REF_PATH="${HEAD_REF#ref: }"; \
-                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
-            fi; \
-        fi)" \
-    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
-    && /usr/bin/uv pip install --no-cache --no-deps "$CODE_DIR" \
-    && rm -f /venv/bin/uv /venv/bin/uvx \
-    && cd / \
-    && ARCHIVEBOX_PY_DIR="$(/venv/bin/python -c 'import pathlib, archivebox; print(pathlib.Path(archivebox.__file__).parent)')" \
-    && /venv/bin/python -m compileall --invalidation-mode checked-hash -q "$ARCHIVEBOX_PY_DIR" \
-    && test -f "$(/venv/bin/python -c 'import archivebox; print(archivebox.__cached__)')" \
-    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
-
-# The builder installs and purges compilers in one layer, so its final
-# filesystem is already runtime-clean. Preserve that ancestry: starting again
-# from archivebox-runtime-base and COPYing /venv would bake a second full venv
-# over the inherited one instead of recording only ArchiveBox's package delta.
-FROM archivebox-builder
+# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
+FROM python:3.11-slim-bookworm

 LABEL name="archivebox" \
    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -217,70 +27,300 @@ LABEL name="archivebox" \
    org.opencontainers.image.vendor="ArchiveBox" \
    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile"
+    com.docker.image.source.entrypoint="Dockerfile" \
+    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
+    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
+    com.docker.desktop.extension.api.version=">= 1.4.7" \
+    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
+    com.docker.extension.publisher-url="https://archivebox.io" \
+    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
+    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
+    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
+    com.docker.extension.categories='database,utility-tools'

-COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
+ARG TARGETPLATFORM
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT

-# The builder invokes abxpkg as root for temporary ELF stripping, which can
-# rewrite derived state ownership. Restore UID 911 before resolving binaries
-# as the runtime user; the old copy-based final stage hid this dependency.
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
-    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
-    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
-    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
-    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
-    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
-    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
-    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
-    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
-    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
-    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
-    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
-    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
-    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$ABXPKG_LIB_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
-    && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+######### Environment Variables #################################

-WORKDIR "$DATA_DIR"
-RUN echo "[+] Initializing image collection..." \
-    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && HOME="$TMP_DIR" archivebox init \
-    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
-        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
-        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
-        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
-        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
-        "$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
-        2>/dev/null || true) \
-    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+# Global system-level config
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    npm_config_loglevel=error

-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+# Version config
+ENV PYTHON_VERSION=3.11 \
+    NODE_VERSION=20

-RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
-    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+# User config
+ENV ARCHIVEBOX_USER="archivebox" \
+    DEFAULT_PUID=911 \
+    DEFAULT_PGID=911

-RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
+# Global paths
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    GLOBAL_VENV=/venv \
+    PLAYWRIGHT_BROWSERS_PATH=/browsers
+
+# Application-level paths
+ENV APP_VENV=/app/.venv \
+    NODE_MODULES=/app/node_modules
+
+# Build shell config
+ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
+
+######### System Environment ####################################
+
+# Detect ArchiveBox version number by reading package.json
+COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
+RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt
+
+# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
+RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
+    && rm -f /etc/apt/apt.conf.d/docker-clean
+
+# Print debug info about build and save it to disk, for human eyes only, not used by anything else
+RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && echo \
+    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
+    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
+    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
+    && echo \
+    && uname -a \
+    && cat /etc/os-release | head -n7 \
+    && which bash && bash --version | head -n1 \
+    && which dpkg && dpkg --version | head -n1 \
+    && echo -e '\n\n' && env && echo -e '\n\n' \
    ) | tee -a /VERSION.txt

+# Create non-privileged user for archivebox and chrome
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
+    && groupadd --system $ARCHIVEBOX_USER \
+    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
+    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
+    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
+    | tee -a /VERSION.txt
+    # DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
+    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
+
+# Install system apt dependencies (adding backports to access more recent apt updates)
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
+    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' > /etc/apt/sources.list.d/backports.list \
+    && mkdir -p /etc/apt/keyrings \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports \
+        # 1. packaging dependencies
+        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
+        # 2. docker and init system dependencies
+        zlib1g-dev dumb-init gosu cron unzip grep \
+        # 3. frivolous CLI helpers to make debugging failed archiving easier
+        # nano iputils-ping dnsutils htop procps jq yq
+    && rm -rf /var/lib/apt/lists/*
+
+######### Language Environments ####################################
+
+# Install Python environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
+    # && apt-get update -qq \
+    # && apt-get install -qq -y -t bookworm-backports --no-upgrade \
+    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip \
+    # && rm -rf /var/lib/apt/lists/* \
+    # tell PDM to allow using global system python site packages
+    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
+    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
+    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
+    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
+    # && python3 -m venv $GLOBAL_VENV \
+    # install global dependencies / python build dependencies in GLOBAL_VENV
+    # && pip install --upgrade pip setuptools wheel \
+    # Save version info
+    && ( \
+        which python3 && python3 --version | grep " $PYTHON_VERSION" \
+        && which pip && pip --version \
+        # && which pdm && pdm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+
+# Install Node environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
+    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-upgrade libatomic1 \
+    && apt-get install -y -t bookworm-backports --no-upgrade \
+        nodejs \
+    && rm -rf /var/lib/apt/lists/* \
+    # Update NPM to latest version
+    && npm i -g npm --cache /root/.npm \
+    # Save version info
+    && ( \
+        which node && node --version \
+        && which npm && npm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+
+######### Extractor Dependencies ##################################
+
+# Install apt dependencies
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing APT extractor dependencies globally using apt..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports \
+        curl wget git yt-dlp ffmpeg ripgrep \
+        # Packages we have also needed in the past:
+        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
+    && rm -rf /var/lib/apt/lists/* \
+    # Save version info
+    && ( \
+        which curl && curl --version | head -n1 \
+        && which wget && wget --version 2>&1 | head -n1 \
+        && which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
+        && which git && git --version 2>&1 | head -n1 \
+        && which rg && rg --version 2>&1 | head -n1 \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install chromium browser using playwright
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports \
+        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+        at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
+        libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
+        libxaw7 libxcomposite1 libxdamage1 libxfont2 \
+        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils xfonts-encodings \
+        # xfonts-scalable xfonts-utils xserver-common xvfb \
+        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
+        # libxss1 dbus dbus-x11 upower \
+    # && service dbus start \
+    # install Chromium using playwright
+    && pip install playwright \
+    && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
+    && playwright install chromium \
+    && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
+    && rm -rf /var/lib/apt/lists/* \
+    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
+    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
+    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
+    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
+    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
+    # Save version info
+    && ( \
+        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install Node dependencies
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
+RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
+    && npm ci --prefer-offline --no-audit --cache /root/.npm \
+    && ( \
+        which node && node --version \
+        && which npm && npm version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+######### Build Dependencies ####################################
+
+# Install ArchiveBox Python dependencies
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports \
+        build-essential \
+        libssl-dev libldap2-dev libsasl2-dev \
+        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
+    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
+    # && pdm use --venv in-project \
+    # && pdm run python -m ensurepip \
+    # && pdm sync --fail-fast --no-editable --group :all --no-self \
+    # && pdm export -o requirements.txt --without-hashes \
+    # && source $GLOBAL_VENV/bin/activate \
+    && pip install -r requirements.txt \
+    && apt-get purge -y \
+        build-essential \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install ArchiveBox Python package from source
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
+    # && apt-get update -qq \
+    # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
+    # && apt-get install -qq -y -t bookworm-backports \
+        # build-essential  \
+    # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
+    && pip install -e "$CODE_DIR"[sonic,ldap] \
+    # save docker image size and always remove compilers / build tools after building is complete
+    # && apt-get purge -y build-essential \
+    # && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+####################################################
+
+# Setup ArchiveBox runtime config
+WORKDIR "$DATA_DIR"
+ENV IN_DOCKER=True \
+    DISPLAY=novnc:0.0 \
+    CUSTOM_TEMPLATES_DIR=/data/templates \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no \
+    ALLOWED_HOSTS=*
+    ## No need to set explicitly, these values will be autodetected by archivebox in docker:
+    # WGET_BINARY="wget" \
+    # YOUTUBEDL_BINARY="yt-dlp" \
+    # CHROME_BINARY="/usr/bin/chromium-browser" \
+    # USE_SINGLEFILE=True \
+    # SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
+    # USE_READABILITY=True \
+    # READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
+    # USE_MERCURY=True \
+    # MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"
+
+# Print version for nice docker finish summary
+RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
+    ) | tee -a /VERSION.txt
+RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
+
+####################################################
+
+# Open up the interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
+    CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'

 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]
+CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]
@@ -1,6 +1,6 @@
 MIT License

-Copyright (c) 2026 Nick Sweeting
+Copyright (c) 2020 Nick Sweeting

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -0,0 +1,34 @@
+# Security Policy
+
+---
+
+## Security Information
+
+Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
+
+https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+
+Also see this section of the README about important caveats when running ArchiveBox:
+
+https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
+
+You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
+
+- https://github.com/ArchiveBox/ArchiveBox#archive-layout
+- https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
+
+---
+
+## Reporting a Vulnerability
+
+We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.
+
+1. Go to the Security tab on our Github repo: https://github.com/ArchiveBox/ArchiveBox/security
+
+2. Click the ["Report a Vulnerability"](https://github.com/ArchiveBox/ArchiveBox/security/advisories/new) button
+
+3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
+
+You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).
@@ -3,4 +3,4 @@ ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E
 select = F,E9,W
 max-line-length = 130
 max-complexity = 10
-exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv,data,data*
+exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv
@@ -0,0 +1 @@
+../LICENSE
@@ -1,150 +1,4 @@
-#!/usr/bin/env python3
-
-# Welcome to the ArchiveBox source code! Thanks for checking it out!
-#
-# "We are swimming upstream against a great torrent of disorganization.
-# In this, our main obligation is to establish arbitrary enclaves of order and system.
-# It is the greatest possible victory to be, to continue to be, and to have been.
-# No defeat can deprive us of the success of having existed for some moment of time
-# in a universe that seems indifferent to us."
-# --Norber Weiner
-
-__package__ = "archivebox"
-
-import os
-import sys
-from pathlib import Path
-from typing import Protocol, cast
+__package__ = 'archivebox'


-class _ReconfigurableStream(Protocol):
-    def reconfigure(self, *, line_buffering: bool) -> object: ...
-
-
-# Force unbuffered output for real-time logs
-if hasattr(sys.stdout, "reconfigure"):
-    cast(_ReconfigurableStream, sys.stdout).reconfigure(line_buffering=True)
-    cast(_ReconfigurableStream, sys.stderr).reconfigure(line_buffering=True)
-os.environ["PYTHONUNBUFFERED"] = "1"
-
-ASCII_LOGO = """
- █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
-██╔══██╗██╔══██╗██╔════╝██║  ██║██║██║   ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
-███████║██████╔╝██║     ███████║██║██║   ██║█████╗   ██████╔╝██║   ██║ ╚███╔╝ 
-██╔══██║██╔══██╗██║     ██╔══██║██║╚██╗ ██╔╝██╔══╝   ██╔══██╗██║   ██║ ██╔██╗ 
-██║  ██║██║  ██║╚██████╗██║  ██║██║ ╚████╔╝ ███████╗ ██████╔╝╚██████╔╝██╔╝ ██╗
-╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
-"""
-
-PACKAGE_DIR = Path(__file__).resolve().parent
-
-os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
-os.environ["TZ"] = "UTC"
-
-# detect ArchiveBox user's UID/GID based on data dir ownership
-from .config.permissions import drop_privileges  # noqa
-
-drop_privileges()
-
-from .misc.checks import check_not_root, check_io_encoding  # noqa
-
-check_not_root()
-check_io_encoding()
-
-from .config.version import VERSION  # noqa
-
-
-__version__ = VERSION
-__author__ = "ArchiveBox"
-__license__ = "MIT"
-
-
-def __getattr__(name: str):
-    if name == "CONSTANTS":
-        from .config.constants import CONSTANTS
-
-        os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
-        return CONSTANTS
-    if name == "DATA_DIR":
-        from .config.paths import DATA_DIR
-
-        return DATA_DIR
-    if name == "VERSION":
-        return VERSION
-    if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
-        from abx_plugins import get_plugins_dir
-        from .config.constants import CONSTANTS
-
-        builtin_plugins_dir = Path(get_plugins_dir()).resolve()
-        user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
-        plugins = {
-            "builtin": builtin_plugins_dir,
-            "user": user_plugins_dir,
-        }
-        values = {
-            "BUILTIN_PLUGINS_DIR": builtin_plugins_dir,
-            "USER_PLUGINS_DIR": user_plugins_dir,
-            "ALL_PLUGINS": plugins,
-            "LOADED_PLUGINS": plugins,
-        }
-        return values[name]
-    raise AttributeError(name)
-
-
-__all__ = (
-    "ASCII_LOGO",
-    "ASCII_ICON",
-    "PACKAGE_DIR",
-    "DATA_DIR",
-    "CONSTANTS",
-    "VERSION",
-    "BUILTIN_PLUGINS_DIR",
-    "USER_PLUGINS_DIR",
-    "ALL_PLUGINS",
-    "LOADED_PLUGINS",
-)
-
-ASCII_ICON = """
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-         ██                                                                            ██          
-         ██                                                                            ██        
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                    ████████████████████████████████████                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ████████████████████████████████████                    ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                 ██████████████████████████████████████████                 ██          
-         ██                 ██████████████████████████████████████████                 ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██        
-         ████████████████████████████████████████████████████████████████████████████████          
-"""
+from .monkey_patches import *
@@ -1,20 +1,11 @@
 #!/usr/bin/env python3
-"""This is the entrypoint for python -m archivebox ..."""

-__package__ = "archivebox"
+__package__ = 'archivebox'

-import archivebox  # noqa # make sure monkey patches are applied before anything else
 import sys

 from .cli import main

-ASCII_LOGO_MINI = r"""
-     _             _     _           ____
-    / \   _ __ ___| |__ (_)_   _____| __ )  _____  __
-   / _ \ | '__/ __| '_ \| \ \ / / _ \  _ \ / _ \ \/ /
-  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  <
- /_/   \_\_|  \___|_| |_|_| \_/ \___|____/ \___/_/\_\
-"""

-if __name__ == "__main__":
-    main(args=sys.argv[1:])
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -0,0 +1 @@
+__package__ = 'abid_utils'
@@ -0,0 +1,202 @@
+from typing import NamedTuple, Any, Union, Optional
+
+import ulid
+import uuid6
+import hashlib
+from urllib.parse import urlparse
+
+from uuid import UUID
+from typeid import TypeID            # type: ignore[import-untyped]
+from datetime import datetime
+
+
+
+ABID_PREFIX_LEN = 4
+ABID_SUFFIX_LEN = 26
+ABID_LEN = 30
+ABID_TS_LEN = 10
+ABID_URI_LEN = 8
+ABID_SUBTYPE_LEN = 2
+ABID_RAND_LEN = 6
+
+DEFAULT_ABID_PREFIX = 'obj_'
+
+# allows people to keep their uris secret on a per-instance basis by changing the salt.
+# the default means everyone can share the same namespace for URI hashes,
+# meaning anyone who has a URI and wants to check if you have it can guess the ABID
+DEFAULT_ABID_URI_SALT = '687c2fff14e3a7780faa5a40c237b19b5b51b089'
+
+
+class ABID(NamedTuple):
+    """
+    e.g. ABID('obj_01HX9FPYTRE4A5CCD901ZYEBQE')
+    """
+    prefix: str            # e.g. obj_
+    ts: str                # e.g. 01HX9FPYTR
+    uri: str               # e.g. E4A5CCD9
+    subtype: str           # e.g. 01
+    rand: str              # e.g. ZYEBQE
+    
+    # salt: str = DEFAULT_ABID_URI_SALT
+
+    def __getattr__(self, attr: str) -> Any:
+        return getattr(self.ulid, attr)
+
+    def __eq__(self, other: Any) -> bool:
+        try:
+            return self.ulid == other.ulid
+        except AttributeError:
+            return NotImplemented
+
+    def __str__(self) -> str:
+        return self.prefix + self.suffix
+
+    def __len__(self) -> int:
+        return len(self.prefix + self.suffix)
+
+    @classmethod
+    def parse(cls, buffer: Union[str, UUID, ulid.ULID, TypeID, 'ABID'], prefix=DEFAULT_ABID_PREFIX) -> 'ABID':
+        assert buffer, f'Attempted to create ABID from null value {buffer}'
+
+        buffer = str(buffer)
+        if '_' in buffer:
+            prefix, suffix = buffer.split('_')
+        else:
+            prefix, suffix = prefix.strip('_'), buffer
+
+        assert len(prefix) == ABID_PREFIX_LEN - 1   # length without trailing _
+        assert len(suffix) == ABID_SUFFIX_LEN, f'Suffix {suffix} from {buffer} was not {ABID_SUFFIX_LEN} chars long'
+
+        return cls(
+            prefix=abid_part_from_prefix(prefix),
+            ts=suffix[0:10].upper(),
+            uri=suffix[10:18].upper(),
+            subtype=suffix[18:20].upper(),
+            rand=suffix[20:26].upper(),
+        )
+    
+    @property
+    def uri_salt(self) -> str:
+        return DEFAULT_ABID_URI_SALT
+
+    @property
+    def suffix(self):
+        return ''.join((self.ts, self.uri, self.subtype, self.rand))
+    
+    @property
+    def ulid(self) -> ulid.ULID:
+        return ulid.parse(self.suffix)
+
+    @property
+    def uuid(self) -> UUID:
+        return self.ulid.uuid
+
+    @property
+    def uuid6(self) -> uuid6.UUID:
+        return uuid6.UUID(hex=self.uuid.hex)
+
+    @property
+    def typeid(self) -> TypeID:
+        return TypeID.from_uuid(prefix=self.prefix.strip('_'), suffix=self.uuid6)
+
+    @property
+    def datetime(self) -> datetime:
+        return self.ulid.timestamp().datetime
+
+
+
+####################################################
+
+
+def uri_hash(uri: Union[str, bytes], salt: str=DEFAULT_ABID_URI_SALT) -> str:
+    """
+    'E4A5CCD9AF4ED2A6E0954DF19FD274E9CDDB4853051F033FD518BFC90AA1AC25'
+    """
+    if isinstance(uri, bytes):
+        uri_str: str = uri.decode()
+    else:
+        uri_str = str(uri)
+
+    # only hash the domain part of URLs
+    if '://' in uri_str:
+        try:
+            domain = urlparse(uri_str).netloc
+            if domain:
+                uri_str = domain
+        except AttributeError:
+            pass
+    
+    uri_bytes = uri_str.encode('utf-8') + salt.encode('utf-8')
+
+    return hashlib.sha256(uri_bytes).hexdigest().upper()
+
+def abid_part_from_prefix(prefix: Optional[str]) -> str:
+    """
+    'snp_'
+    """
+    if prefix is None:
+        return 'obj_'
+
+    prefix = prefix.strip('_').lower()
+    assert len(prefix) == 3
+    return prefix + '_'
+
+def abid_part_from_uri(uri: str, salt: str=DEFAULT_ABID_URI_SALT) -> str:
+    """
+    'E4A5CCD9'     # takes first 8 characters of sha256(url)
+    """
+    uri = str(uri)
+    return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
+
+def abid_part_from_ts(ts: Optional[datetime]) -> str:
+    """
+    '01HX9FPYTR'   # produces 10 character Timestamp section of ulid based on added date
+    """
+    return str(ulid.from_timestamp(ts) if ts else ulid.new())[:ABID_TS_LEN]
+
+def abid_part_from_subtype(subtype: str) -> str:
+    """
+    Snapshots have 01 type, other objects have other subtypes like wget/media/etc.
+    Also allows us to change the ulid spec later by putting special sigil values here.
+    """
+    subtype = str(subtype)
+    if len(subtype) == ABID_SUBTYPE_LEN:
+        return subtype
+
+    return hashlib.sha256(subtype.encode('utf-8')).hexdigest()[:ABID_SUBTYPE_LEN].upper()
+
+def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
+    """
+    'ZYEBQE'   # takes last 6 characters of randomness from existing legacy uuid db field
+    """
+    if rand is None:
+        # if it's None we generate a new random 6 character hex string
+        return str(ulid.new())[-ABID_RAND_LEN:]
+    elif isinstance(rand, UUID):
+        # if it's a uuid we take the last 6 characters of the ULID represation of it
+        return str(ulid.from_uuid(rand))[-ABID_RAND_LEN:]
+    elif isinstance(rand, int):
+        # if it's a BigAutoInteger field we convert it from an int to a 0-padded string
+        rand_str = str(rand)[-ABID_RAND_LEN:]
+        padding_needed = ABID_RAND_LEN - len(rand_str)
+        rand_str = ('0'*padding_needed) + rand_str
+        return rand_str
+
+    # otherwise treat it as a string, take the last 6 characters of it verbatim
+    return str(rand)[-ABID_RAND_LEN:].upper()
+
+
+def abid_from_values(prefix, ts, uri, subtype, rand, salt=DEFAULT_ABID_URI_SALT) -> ABID:
+    """
+    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+    """
+
+    abid = ABID(
+        prefix=abid_part_from_prefix(prefix),
+        ts=abid_part_from_ts(ts),
+        uri=abid_part_from_uri(uri, salt=salt),
+        subtype=abid_part_from_subtype(subtype),
+        rand=abid_part_from_rand(rand),
+    )
+    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for ts={ts} uri={uri} subtyp={subtype} rand={rand}'
+    return abid
@@ -0,0 +1,18 @@
+from django.contrib import admin
+
+
+class ABIDModelAdmin(admin.ModelAdmin):
+    list_display = ('created', 'created_by', 'abid', '__str__')
+    sort_fields = ('created', 'created_by', 'abid', '__str__')
+    readonly_fields = ('abid', 'created', '__str__')
+
+    def get_form(self, request, obj=None, **kwargs):
+        form = super().get_form(request, obj, **kwargs)
+        if 'created_by' in form.base_fields:
+            form.base_fields['created_by'].initial = request.user
+        return form
+
+    # def save_model(self, request, obj, form, change):
+    #     if getattr(obj, 'created_by_id', None) in (None, get_or_create_system_user_pk()):
+    #         obj.created_by = request.user
+    #     obj.save()
@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class AbidUtilsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    
+    name = 'abid_utils'
@@ -0,0 +1,344 @@
+"""
+This file provides the Django ABIDField and ABIDModel base model to inherit from.
+
+It implements the ArchiveBox ID (ABID) interfaces including abid_values, generate_abid, .abid, .uuid, .id.
+"""
+
+from typing import Any, Dict, Union, List, Set, NamedTuple, cast
+
+from ulid import ULID
+from uuid import uuid4, UUID
+from typeid import TypeID            # type: ignore[import-untyped]
+from datetime import datetime
+from functools import partial
+from charidfield import CharIDField  # type: ignore[import-untyped]
+
+from django.conf import settings
+from django.db import models
+from django.utils import timezone
+from django.db.utils import OperationalError
+from django.contrib.auth import get_user_model
+
+from django_stubs_ext.db.models import TypedModelMeta
+
+from .abid import (
+    ABID,
+    ABID_LEN,
+    ABID_RAND_LEN,
+    ABID_SUFFIX_LEN,
+    DEFAULT_ABID_PREFIX,
+    DEFAULT_ABID_URI_SALT,
+    abid_part_from_prefix,
+    abid_from_values
+)
+
+####################################################
+
+
+# Database Field for typeid/ulid style IDs with a prefix, e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ
+ABIDField = partial(
+    CharIDField,
+    max_length=ABID_LEN,
+    help_text="ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)",
+    default=None,
+    null=True,
+    blank=True,
+    db_index=True,
+    unique=True,
+)
+
+def get_or_create_system_user_pk(username='system'):
+    """Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
+
+    User = get_user_model()
+
+    # if only one user exists total, return that user
+    if User.objects.filter(is_superuser=True).count() == 1:
+        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
+
+    # otherwise, create a dedicated "system" user
+    user, created = User.objects.get_or_create(username=username, is_staff=True, is_superuser=True, defaults={'email': '', 'password': ''})
+    return user.pk
+
+
+class ABIDModel(models.Model):
+    """
+    Abstract Base Model for other models to depend on. Provides ArchiveBox ID (ABID) interface.
+    """
+    abid_prefix: str = DEFAULT_ABID_PREFIX  # e.g. 'tag_'
+    abid_ts_src = 'None'                    # e.g. 'self.created'
+    abid_uri_src = 'None'                   # e.g. 'self.uri'
+    abid_subtype_src = 'None'               # e.g. 'self.extractor'
+    abid_rand_src = 'None'                  # e.g. 'self.uuid' or 'self.id'
+
+    # id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
+    # uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    def save(self, *args: Any, **kwargs: Any) -> None:
+        # when first creating a row, self.ABID is the source of truth
+        # overwrite default prefilled self.id & self.abid with generated self.ABID value
+        if self._state.adding or not self.id:
+            self.id = self.ABID.uuid
+        if self._state.adding or not self.abid:
+            self.abid = str(self.ABID)
+
+        super().save(*args, **kwargs)
+        assert str(self.id) == str(self.ABID.uuid), f'self.id {self.id} does not match self.ABID {self.ABID.uuid}'
+        assert str(self.abid) == str(self.ABID), f'self.abid {self.id} does not match self.ABID {self.ABID.uuid}'
+
+    @property
+    def abid_values(self) -> Dict[str, Any]:
+        return {
+            'prefix': self.abid_prefix,
+            'ts': eval(self.abid_ts_src),
+            'uri': eval(self.abid_uri_src),
+            'subtype': eval(self.abid_subtype_src),
+            'rand': eval(self.abid_rand_src),
+        }
+
+    def generate_abid(self) -> ABID:
+        """
+        Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+        """
+        prefix, ts, uri, subtype, rand = self.abid_values.values()
+
+        if (not prefix) or prefix == DEFAULT_ABID_PREFIX:
+            suggested_abid = self.__class__.__name__[:3].lower()
+            raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
+
+        if not ts:
+            # default to unix epoch with 00:00:00 UTC
+            ts = datetime.fromtimestamp(0, timezone.utc)     # equivalent to: ts = datetime.utcfromtimestamp(0)
+            print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
+
+        if not uri:
+            uri = str(self)
+            print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
+
+        if not subtype:
+            subtype = self.__class__.__name__
+            print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
+
+        if not rand:
+            rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
+            print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
+
+        abid = abid_from_values(
+            prefix=prefix,
+            ts=ts,
+            uri=uri,
+            subtype=subtype,
+            rand=rand,
+            salt=DEFAULT_ABID_URI_SALT,
+        )
+        assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
+        return abid
+
+    @property
+    def ABID(self) -> ABID:
+        """
+        ULIDParts(timestamp='01HX9FPYTR', url='E4A5CCD9', subtype='00', randomness='ZYEBQE')
+        """
+        
+        # if object is not yet saved to DB, always generate fresh ABID from values
+        if self._state.adding:
+            return self.generate_abid()
+        
+        # otherwise DB is single source of truth, load ABID from existing db pk
+        abid: ABID | None = None
+        try:
+            abid = abid or ABID.parse(self.pk)
+        except Exception:
+            pass
+
+        try:
+            abid = abid or ABID.parse(self.id)
+        except Exception:
+            pass
+
+        try:
+            abid = abid or ABID.parse(cast(str, self.abid))
+        except Exception:
+            pass
+
+        abid = abid or self.generate_abid()
+
+        return abid
+
+    @property
+    def ULID(self) -> ULID:
+        """
+        Get a ulid.ULID representation of the object's ABID.
+        """
+        return self.ABID.ulid
+
+    @property
+    def UUID(self) -> UUID:
+        """
+        Get a uuid.UUID (v4) representation of the object's ABID.
+        """
+        return self.ABID.uuid
+
+    @property
+    def TypeID(self) -> TypeID:
+        """
+        Get a typeid.TypeID (stripe-style) representation of the object's ABID.
+        """
+        return self.ABID.typeid
+
+
+
+####################################################
+
+# Django helpers
+def find_all_abid_prefixes() -> Dict[str, type[models.Model]]:
+    """
+    Return the mapping of all ABID prefixes to their models.
+    e.g. {'tag_': core.models.Tag, 'snp_': core.models.Snapshot, ...}
+    """
+    import django.apps
+    prefix_map = {}
+
+    for model in django.apps.apps.get_models():
+        abid_prefix = getattr(model, 'abid_prefix', None)
+        if abid_prefix:
+            prefix_map[abid_prefix] = model
+    return prefix_map
+
+def find_prefix_for_abid(abid: ABID) -> str:
+    """
+    Find the correct prefix for a given ABID that may have be missing a prefix (slow).
+    e.g. ABID('obj_01BJQMF54D093DXEAWZ6JYRPAQ') -> 'snp_'
+    """
+    # if existing abid prefix is correct, lookup is easy
+    model = find_model_from_abid(abid)
+    if model:
+        assert issubclass(model, ABIDModel)
+        return model.abid_prefix
+
+    # prefix might be obj_ or missing, fuzzy-search to find any object that matches
+    return find_obj_from_abid_rand(abid)[0].abid_prefix
+
+def find_model_from_abid_prefix(prefix: str) -> type[ABIDModel] | None:
+    """
+    Return the Django Model that corresponds to a given ABID prefix.
+    e.g. 'tag_' -> core.models.Tag
+    """
+    prefix = abid_part_from_prefix(prefix)
+
+    import django.apps
+
+    for model in django.apps.apps.get_models():
+        if not issubclass(model, ABIDModel): continue   # skip non-ABID-enabled models
+        if not hasattr(model, 'objects'): continue      # skip abstract models
+
+        if (model.abid_prefix == prefix):
+            return model
+
+    return None
+
+def find_model_from_abid(abid: ABID) -> type[models.Model] | None:
+    """
+    Shortcut for find_model_from_abid_prefix(abid.prefix)
+    """
+    return find_model_from_abid_prefix(abid.prefix)
+
+def find_obj_from_abid_rand(rand: Union[ABID, str], model=None) -> List[ABIDModel]:
+    """
+    Find an object corresponding to an ABID by exhaustively searching using its random suffix (slow).
+    e.g. 'obj_....................JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
+    """
+
+    # convert str to ABID if necessary
+    if isinstance(rand, ABID):
+        abid: ABID = rand
+    else:
+        rand = str(rand)
+        if len(rand) < ABID_SUFFIX_LEN:
+            padding_needed = ABID_SUFFIX_LEN - len(rand)
+            rand = ('0'*padding_needed) + rand
+        abid = ABID.parse(rand)
+
+    import django.apps
+
+    partial_matches: List[ABIDModel] = []
+
+    models_to_try = cast(Set[type[models.Model]], set(filter(bool, (
+        model,
+        find_model_from_abid(abid),
+        *django.apps.apps.get_models(),
+    ))))
+    # print(abid, abid.rand, abid.uuid, models_to_try)
+
+    for model in models_to_try:
+        if not issubclass(model, ABIDModel): continue   # skip Models that arent ABID-enabled
+        if not hasattr(model, 'objects'): continue      # skip abstract Models
+        assert hasattr(model, 'objects')                # force-fix for type hint nit about missing manager https://github.com/typeddjango/django-stubs/issues/1684
+
+        # continue on to try fuzzy searching by randomness portion derived from uuid field
+        try:
+            qs = []
+            if hasattr(model, 'abid'):
+                qs = model.objects.filter(abid__endswith=abid.rand)
+            elif hasattr(model, 'uuid'):
+                qs = model.objects.filter(uuid__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
+            elif hasattr(model, 'id'):
+                # NOTE: this only works on SQLite where every column is a string
+                # other DB backends like postgres dont let you do __endswith if this is a BigAutoInteger field
+                
+                # try to search for uuid=...-2354352
+                # try to search for id=...2354352
+                # try to search for id=2354352
+                qs = model.objects.filter(
+                    models.Q(id__endswith=str(abid.uuid)[-ABID_RAND_LEN:])
+                    | models.Q(id__endswith=abid.rand)
+                    | models.Q(id__startswith=str(int(abid.rand)) if abid.rand.isdigit() else abid.rand)
+                )
+
+            for obj in qs:
+                if obj.generate_abid() == abid:
+                    # found exact match, no need to keep iterating
+                    return [obj]
+                partial_matches.append(obj)
+        except OperationalError as err:
+            print(f'[!] WARNING: Got error while trying to iterate through QuerySet for {model}:', err, '\n')
+
+    return partial_matches
+
+def find_obj_from_abid(abid: ABID, model=None, fuzzy=False) -> Any:
+    """
+    Find an object with a given ABID by filtering possible models for a matching abid/uuid/id (fast).
+    e.g. 'snp_01BJQMF54D093DXEAWZ6JYRPAQ' -> Snapshot('snp_01BJQMF54D093DXEAWZ6JYRPAQ')
+    """
+
+    model = model or find_model_from_abid(abid)
+    assert model, f'Could not find model that could match this ABID type: {abid}'
+
+    try:
+        if hasattr(model, 'abid'):
+            return model.objects.get(abid__endswith=abid.suffix)
+        if hasattr(model, 'uuid'):
+            return model.objects.get(uuid=abid.uuid)
+        return model.objects.get(id=abid.uuid)
+    except model.DoesNotExist:
+        # if the model has an abid field then it shouldve matched, pointless to fuzzy search in that case
+        if hasattr(model, 'abid') or (not fuzzy):
+            raise
+
+    # continue on to try fuzzy searching by randomness portion derived from uuid field
+    match_by_rand = find_obj_from_abid_rand(abid, model=model)
+    if match_by_rand:
+        if match_by_rand[0].abid_prefix != abid.prefix:
+            print(f'[!] WARNING: fetched object {match_by_rand} even though prefix {abid.prefix} doesnt match!', abid, '\n')
+        return match_by_rand
+
+    raise model.DoesNotExist
+
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
@@ -1 +1 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'
@@ -1,72 +0,0 @@
-__package__ = "archivebox.api"
-
-from django import forms
-from django.contrib import admin
-from django.http import HttpRequest
-from django.utils.text import capfirst
-from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
-from signal_webhooks.settings import webhook_settings
-from signal_webhooks.utils import get_webhook_model, model_from_reference
-
-from archivebox.base_models.admin import card_fieldset, BaseModelAdmin
-
-from archivebox.api.models import APIToken
-
-
-def _webhook_fields(*names: str) -> tuple[str, ...]:
-    model_fields = {field.name for field in get_webhook_model()._meta.fields}
-    return tuple(name for name in names if name in model_fields)
-
-
-class APITokenAdmin(BaseModelAdmin):
-    list_display = ("created_at", "id", "created_by", "token_redacted", "expires")
-    sort_fields = ("id", "created_at", "created_by", "expires")
-    readonly_fields = ("created_at", "modified_at")
-    search_fields = ("id", "created_by__username", "token")
-
-    fieldsets = (
-        card_fieldset("Token", ("token", "expires")),
-        card_fieldset("Owner", ("created_by",)),
-        card_fieldset("Timestamps", ("created_at", "modified_at")),
-    )
-
-    list_filter = ("created_by",)
-    ordering = ["-created_at"]
-    list_per_page = 100
-
-
-class OutboundWebhookAdminForm(WebhookModelForm):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.fields["ref"] = forms.ChoiceField(
-            label=self.fields["ref"].label,
-            help_text=self.fields["ref"].help_text,
-            choices=[
-                (ref, f"{capfirst(model_from_reference(ref, check_hooks=False)._meta.verbose_name_plural)} ({ref})")
-                for ref in sorted(webhook_settings.HOOKS)
-            ],
-        )
-
-
-class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
-    form = OutboundWebhookAdminForm
-    list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
-    sort_fields = _webhook_fields("created_at", "created_by", "id", "ref", "endpoint", "last_success", "last_failure")
-    readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
-
-    fieldsets = (
-        card_fieldset("Webhook", _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"), wide=True),
-        card_fieldset("Authentication", _webhook_fields("auth_token")),
-        card_fieldset("Status", _webhook_fields("enabled", "last_success", "last_failure", "last_response")),
-        card_fieldset("Owner", _webhook_fields("created_by")),
-        card_fieldset("Timestamps", _webhook_fields("created_at", "modified_at")),
-    )
-
-    def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:
-        """Preserve WebhookAdmin's auth token filter with Django's current admin signature."""
-        return not lookup.startswith("auth_token") and admin.ModelAdmin.lookup_allowed(self, lookup, value, request)
-
-
-def register_admin(admin_site: admin.AdminSite) -> None:
-    admin_site.register(APIToken, APITokenAdmin)
-    admin_site.register(get_webhook_model(), CustomWebhookAdmin)
@@ -1,14 +1,11 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

 from django.apps import AppConfig


+
 class APIConfig(AppConfig):
-    name = "archivebox.api"
-    label = "api"
+    name = 'api'

-
-def register_admin(admin_site):
-    from archivebox.api.admin import register_admin
-
-    register_admin(admin_site)
+    def ready(self):
+        pass
@@ -1,130 +1,102 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from datetime import timedelta
+from typing import Optional, cast

-from django.utils import timezone
 from django.http import HttpRequest
+from django.contrib.auth import login
 from django.contrib.auth import authenticate
-from django.contrib.auth.models import User
+from django.contrib.auth.models import AbstractBaseUser

-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader
-from ninja.errors import HttpError
+from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth, django_auth_superuser


-def get_or_create_api_token(user: User | None):
-    from archivebox.api.models import APIToken
-
-    if user and user.is_superuser:
-        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
-        if api_tokens.exists():
-            # unexpired token exists, use it
-            api_token = api_tokens.last()
-        else:
-            # does not exist, create a new one
-            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
-
-        if api_token is None:
-            return None
-        assert api_token.is_valid(), f"API token is not valid {api_token}"
-
-        return api_token
-    return None
-
-
-def auth_using_token(token: str | None, request: HttpRequest | None = None) -> User | None:
+def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
    """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
-    from archivebox.api.models import APIToken  # lazy import model to avoid loading it at urls.py import time
+    from api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
+    
+    user = None

-    user: User | None = None
-
-    submitted_empty_form = str(token).strip() in ("string", "", "None", "null")
-    if not submitted_empty_form:
+    submitted_empty_form = token in ('string', '', None)
+    if submitted_empty_form:
+        assert request is not None, 'No request provided for API key authentication'
+        user = request.user       # see if user is authed via django session and use that as the default
+    else:
        try:
-            api_token = APIToken.objects.get(token=token)
-            if api_token.is_valid() and isinstance(api_token.created_by, User):
-                user = api_token.created_by
-                if request is not None:
-                    setattr(request, "_api_token", api_token)
+            token = APIToken.objects.get(token=token)
+            if token.is_valid():
+                user = token.created_by
        except APIToken.DoesNotExist:
            pass

-    return user
+    if not user:
+        print('[❌] Failed to authenticate API user using API Key:', request)

-
-def token_from_request(request: HttpRequest) -> str:
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-    return token
-
-
-def authenticated_user_from_request(request: HttpRequest) -> User | None:
-    user = request.user
-    if user.is_authenticated and user.is_active:
-        return user
-
-    token = token_from_request(request)
-    token_user = auth_using_token(token=token, request=request) if token else None
-    if token_user and token_user.is_active:
-        request.user = token_user
-        return token_user
    return None

-
-def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
+def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
    """Given a username and password, check if they are valid and return the corresponding user"""
-    user: User | None = None
-
-    submitted_empty_form = (username, password) in (("string", "string"), ("", ""), (None, None))
-    if not submitted_empty_form:
-        authenticated_user = authenticate(
+    user = None
+    
+    submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
+    if submitted_empty_form:
+        assert request is not None, 'No request provided for API key authentication'
+        user = request.user       # see if user is authed via django session and use that as the default
+    else:
+        user = authenticate(
            username=username,
            password=password,
        )
-        if isinstance(authenticated_user, User):
-            user = authenticated_user
-    return user
+
+    if not user:
+        print('[❌] Failed to authenticate API user using API Key:', request)
+        user = None
+
+    return cast(AbstractBaseUser | None, user)


 ### Base Auth Types

+class APITokenAuthCheck:
+    """The base class for authentication methods that use an api.models.APIToken"""
+    def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
+        user = auth_using_token(
+            token=key,
+            request=request,
+        )
+        if user is not None:
+            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
+        return user

-def _require_superuser(user: User | None, request: HttpRequest, auth_method: str) -> User | None:
-    if user and user.pk:
-        request.user = user
-        setattr(request, "_api_auth_method", auth_method)
-        if not user.is_superuser:
-            raise HttpError(403, "Valid credentials but User does not have permission (make sure user.is_superuser=True)")
-    return user
+class UserPassAuthCheck:
+    """The base class for authentication methods that use a username & password"""
+    def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
+        user = auth_using_password(
+            username=username,
+            password=password,
+            request=request,
+        )
+        if user is not None:
+            login(request, user, backend='django.contrib.auth.backends.ModelBackend')
+        return user


 ### Django-Ninja-Provided Auth Methods

-
-class HeaderTokenAuth(APIKeyHeader):
+class HeaderTokenAuth(APITokenAuthCheck, APIKeyHeader):
    """Allow authenticating by passing X-API-Key=xyz as a request header"""
-
    param_name = "X-ArchiveBox-API-Key"

-    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
-        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
-
-
-class BearerTokenAuth(HttpBearer):
+class BearerTokenAuth(APITokenAuthCheck, HttpBearer):
    """Allow authenticating by passing Bearer=xyz as a request header"""
+    pass

-    def authenticate(self, request: HttpRequest, token: str) -> User | None:
-        return _require_superuser(auth_using_token(token=token, request=request), request, self.__class__.__name__)
-
-
-class QueryParamTokenAuth(APIKeyQuery):
+class QueryParamTokenAuth(APITokenAuthCheck, APIKeyQuery):
    """Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
-
    param_name = "api_key"

-    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
-        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
+class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
+    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
+    pass


 ### Enabled Auth Methods
@@ -132,6 +104,7 @@ class QueryParamTokenAuth(APIKeyQuery):
 API_AUTH_METHODS = [
    HeaderTokenAuth(),
    BearerTokenAuth(),
-    QueryParamTokenAuth(),
-    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
+    QueryParamTokenAuth(), 
+    django_auth_superuser,
+    UsernameAndPasswordAuth(),
 ]
@@ -1,32 +0,0 @@
-__package__ = "archivebox.api"
-
-from django.http import HttpResponse
-
-
-class ApiCorsMiddleware:
-    """Attach permissive CORS headers for API routes (token-based auth)."""
-
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        if request.path.startswith("/api/"):
-            if request.method == "OPTIONS" and request.META.get("HTTP_ACCESS_CONTROL_REQUEST_METHOD"):
-                response = HttpResponse(status=204)
-                return self._add_cors_headers(request, response)
-
-            response = self.get_response(request)
-            return self._add_cors_headers(request, response)
-
-        return self.get_response(request)
-
-    def _add_cors_headers(self, request, response):
-        origin = request.META.get("HTTP_ORIGIN")
-        if not origin:
-            return response
-
-        response["Access-Control-Allow-Origin"] = "*"
-        response["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
-        response["Access-Control-Allow-Headers"] = "Authorization, X-ArchiveBox-API-Key, Content-Type, X-CSRFToken"
-        response["Access-Control-Max-Age"] = "600"
-        return response
@@ -1,281 +1,29 @@
-# Generated by hand on 2025-12-29
-# Creates APIToken and OutboundWebhook tables using raw SQL
+# Generated by Django 4.2.11 on 2024-04-25 04:19

+import api.models
+from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
-import django.utils.timezone
-from django.conf import settings
-from archivebox.uuid_compat import uuid7
-from archivebox.base_models.models import get_or_create_system_user_pk
-import archivebox.api.models
-import signal_webhooks.fields
-import signal_webhooks.utils
-
-
-# Raw sqlite DDL kept EXACTLY as-is; executed only on sqlite via Django's
-# RunSQL (identical statement splitting → byte-for-byte identical behavior).
-INITIAL_SQL = """
-                -- Create api_apitoken table
-                CREATE TABLE IF NOT EXISTS api_apitoken (
-                    id TEXT PRIMARY KEY NOT NULL,
-                    created_at DATETIME NOT NULL,
-                    modified_at DATETIME NOT NULL,
-
-                    token VARCHAR(32) NOT NULL UNIQUE,
-                    expires DATETIME,
-
-                    created_by_id INTEGER NOT NULL,
-
-                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
-                );
-                CREATE INDEX IF NOT EXISTS api_apitoken_created_by_id_idx ON api_apitoken(created_by_id);
-                CREATE INDEX IF NOT EXISTS api_apitoken_created_at_idx ON api_apitoken(created_at);
-                CREATE INDEX IF NOT EXISTS api_apitoken_token_idx ON api_apitoken(token);
-
-                -- Create api_outboundwebhook table
-                CREATE TABLE IF NOT EXISTS api_outboundwebhook (
-                    id TEXT PRIMARY KEY NOT NULL,
-                    created_at DATETIME NOT NULL,
-                    modified_at DATETIME NOT NULL,
-
-                    name VARCHAR(255) NOT NULL UNIQUE,
-                    signal VARCHAR(255) NOT NULL,
-                    ref VARCHAR(1024) NOT NULL,
-                    endpoint VARCHAR(2048) NOT NULL,
-                    headers TEXT NOT NULL DEFAULT '{}',
-                    auth_token TEXT NOT NULL DEFAULT '',
-                    enabled BOOLEAN NOT NULL DEFAULT 1,
-                    keep_last_response BOOLEAN NOT NULL DEFAULT 0,
-                    created DATETIME NOT NULL,
-                    updated DATETIME NOT NULL,
-                    last_response TEXT NOT NULL DEFAULT '',
-                    last_success DATETIME,
-                    last_failure DATETIME,
-
-                    created_by_id INTEGER NOT NULL,
-
-                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
-                );
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_by_id_idx ON api_outboundwebhook(created_by_id);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
-                    """
-
-INITIAL_REVERSE_SQL = """
-                DROP TABLE IF EXISTS api_outboundwebhook;
-                DROP TABLE IF EXISTS api_apitoken;
-                    """
-
-
-def _run_sqlite_only_sql(apps, schema_editor):
-    if schema_editor.connection.vendor != "sqlite":
-        return
-    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_forwards(
-        "api",
-        schema_editor,
-        None,
-        None,
-    )
-
-
-def _run_sqlite_only_sql_reverse(apps, schema_editor):
-    if schema_editor.connection.vendor != "sqlite":
-        return
-    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_backwards(
-        "api",
-        schema_editor,
-        None,
-        None,
-    )
-
-
-def _pg_sync_schema(apps, schema_editor):
-    from archivebox.misc.db import rebuild_models_from_migration_state
-
-    rebuild_models_from_migration_state(apps, schema_editor, "api", ["APIToken", "OutboundWebhook"])
-
-
-def _pg_drop_schema(apps, schema_editor):
-    from archivebox.misc.db import drop_models_on_postgres
-
-    drop_models_on_postgres(apps, schema_editor, "api", ["OutboundWebhook", "APIToken"])
+import uuid


 class Migration(migrations.Migration):
+
    initial = True

    dependencies = [
-        ("auth", "0012_alter_user_first_name_max_length"),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
-        migrations.SeparateDatabaseAndState(
-            database_operations=[
-                migrations.RunPython(_run_sqlite_only_sql, reverse_code=_run_sqlite_only_sql_reverse),
-            ],
-            state_operations=[
-                migrations.CreateModel(
-                    name="APIToken",
-                    fields=[
-                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
-                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
-                        ("modified_at", models.DateTimeField(auto_now=True)),
-                        ("token", models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
-                        ("expires", models.DateTimeField(blank=True, null=True)),
-                        (
-                            "created_by",
-                            models.ForeignKey(
-                                default=get_or_create_system_user_pk,
-                                on_delete=django.db.models.deletion.CASCADE,
-                                to=settings.AUTH_USER_MODEL,
-                            ),
-                        ),
-                    ],
-                    options={
-                        "verbose_name": "API Key",
-                        "verbose_name_plural": "API Keys",
-                        "app_label": "api",
-                    },
-                ),
-                migrations.CreateModel(
-                    name="OutboundWebhook",
-                    fields=[
-                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
-                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
-                        ("modified_at", models.DateTimeField(auto_now=True)),
-                        (
-                            "name",
-                            models.CharField(db_index=True, help_text="Webhook name.", max_length=255, unique=True, verbose_name="name"),
-                        ),
-                        (
-                            "signal",
-                            models.CharField(
-                                choices=[
-                                    ("CREATE", "Create"),
-                                    ("UPDATE", "Update"),
-                                    ("DELETE", "Delete"),
-                                    ("M2M", "M2M changed"),
-                                    ("CREATE_OR_UPDATE", "Create or Update"),
-                                    ("CREATE_OR_DELETE", "Create or Delete"),
-                                    ("CREATE_OR_M2M", "Create or M2M changed"),
-                                    ("UPDATE_OR_DELETE", "Update or Delete"),
-                                    ("UPDATE_OR_M2M", "Update or M2M changed"),
-                                    ("DELETE_OR_M2M", "Delete or M2M changed"),
-                                    ("CREATE_UPDATE_OR_DELETE", "Create, Update or Delete"),
-                                    ("CREATE_UPDATE_OR_M2M", "Create, Update or M2M changed"),
-                                    ("CREATE_DELETE_OR_M2M", "Create, Delete or M2M changed"),
-                                    ("UPDATE_DELETE_OR_M2M", "Update, Delete or M2M changed"),
-                                    ("CREATE_UPDATE_DELETE_OR_M2M", "Create, Update or Delete, or M2M changed"),
-                                ],
-                                help_text="Signal the webhook fires to.",
-                                max_length=255,
-                                verbose_name="signal",
-                            ),
-                        ),
-                        (
-                            "ref",
-                            models.CharField(
-                                db_index=True,
-                                help_text="Dot import notation to the model the webhook is for.",
-                                max_length=1023,
-                                validators=[signal_webhooks.utils.model_from_reference],
-                                verbose_name="referenced model",
-                            ),
-                        ),
-                        (
-                            "endpoint",
-                            models.URLField(help_text="Target endpoint for this webhook.", max_length=2047, verbose_name="endpoint"),
-                        ),
-                        (
-                            "headers",
-                            models.JSONField(
-                                blank=True,
-                                default=dict,
-                                help_text="Headers to send with the webhook request.",
-                                validators=[signal_webhooks.utils.is_dict],
-                                verbose_name="headers",
-                            ),
-                        ),
-                        (
-                            "auth_token",
-                            signal_webhooks.fields.TokenField(
-                                blank=True,
-                                default="",
-                                help_text="Authentication token to use in an Authorization header.",
-                                max_length=8000,
-                                validators=[signal_webhooks.utils.decode_cipher_key],
-                                verbose_name="authentication token",
-                            ),
-                        ),
-                        ("enabled", models.BooleanField(default=True, help_text="Is this webhook enabled?", verbose_name="enabled")),
-                        (
-                            "keep_last_response",
-                            models.BooleanField(
-                                default=False,
-                                help_text="Should the webhook keep a log of the latest response it got?",
-                                verbose_name="keep last response",
-                            ),
-                        ),
-                        (
-                            "created",
-                            models.DateTimeField(auto_now_add=True, help_text="When the webhook was created.", verbose_name="created"),
-                        ),
-                        (
-                            "updated",
-                            models.DateTimeField(auto_now=True, help_text="When the webhook was last updated.", verbose_name="updated"),
-                        ),
-                        (
-                            "last_response",
-                            models.CharField(
-                                blank=True,
-                                default="",
-                                help_text="Latest response to this webhook.",
-                                max_length=8000,
-                                verbose_name="last response",
-                            ),
-                        ),
-                        (
-                            "last_success",
-                            models.DateTimeField(
-                                default=None,
-                                help_text="When the webhook last succeeded.",
-                                null=True,
-                                verbose_name="last success",
-                            ),
-                        ),
-                        (
-                            "last_failure",
-                            models.DateTimeField(
-                                default=None,
-                                help_text="When the webhook last failed.",
-                                null=True,
-                                verbose_name="last failure",
-                            ),
-                        ),
-                        (
-                            "created_by",
-                            models.ForeignKey(
-                                default=get_or_create_system_user_pk,
-                                on_delete=django.db.models.deletion.CASCADE,
-                                to=settings.AUTH_USER_MODEL,
-                            ),
-                        ),
-                    ],
-                    options={
-                        "verbose_name": "API Outbound Webhook",
-                        "app_label": "api",
-                    },
-                ),
-                migrations.AddConstraint(
-                    model_name="outboundwebhook",
-                    constraint=models.UniqueConstraint(fields=["ref", "endpoint"], name="prevent_duplicate_hooks_api_outboundwebhook"),
-                ),
+        migrations.CreateModel(
+            name='APIToken',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('token', models.CharField(default=api.models.generate_secret_token, max_length=32, unique=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('expires', models.DateTimeField(blank=True, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
            ],
        ),
-        # On non-sqlite backends the raw DDL above is skipped, so the real
-        # schema diverges from Django state. Rebuild both tables (including the
-        # unique constraint) from the post-migration state (no-op on sqlite).
-        migrations.RunPython(_pg_sync_schema, reverse_code=_pg_drop_schema),
    ]
@@ -0,0 +1,17 @@
+# Generated by Django 5.0.4 on 2024-04-26 05:28
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='apitoken',
+            options={'verbose_name': 'API Key', 'verbose_name_plural': 'API Keys'},
+        ),
+    ]
@@ -0,0 +1,77 @@
+# Generated by Django 5.0.6 on 2024-06-03 01:52
+
+import abid_utils.models
+import charidfield.fields
+import django.db.models.deletion
+import signal_webhooks.fields
+import signal_webhooks.utils
+import uuid
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0002_alter_apitoken_options'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='apitoken',
+            old_name='user',
+            new_name='created_by',
+        ),
+        migrations.AddField(
+            model_name='apitoken',
+            name='abid',
+            field=charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='apt_', unique=True),
+        ),
+        migrations.AddField(
+            model_name='apitoken',
+            name='modified',
+            field=models.DateTimeField(auto_now=True),
+        ),
+        migrations.AddField(
+            model_name='apitoken',
+            name='uuid',
+            field=models.UUIDField(blank=True, null=True, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False),
+        ),
+        migrations.CreateModel(
+            name='OutboundWebhook',
+            fields=[
+                ('name', models.CharField(db_index=True, help_text='Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).', max_length=255, unique=True, verbose_name='name')),
+                ('signal', models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='The type of event the webhook should fire for (e.g. Create, Update, Delete).', max_length=255, verbose_name='signal')),
+                ('ref', models.CharField(db_index=True, help_text='Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model')),
+                ('endpoint', models.URLField(help_text='External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).', max_length=2047, verbose_name='endpoint')),
+                ('headers', models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers')),
+                ('auth_token', signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token')),
+                ('enabled', models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled')),
+                ('keep_last_response', models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response')),
+                ('updated', models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated')),
+                ('last_response', models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response')),
+                ('last_success', models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success')),
+                ('last_failure', models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('modified', models.DateTimeField(auto_now=True)),
+                ('id', models.UUIDField(blank=True, null=True, unique=True)),
+                ('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='whk_', unique=True)),
+                ('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'verbose_name': 'API Outbound Webhook',
+                'abstract': False,
+            },
+        ),
+        migrations.AddConstraint(
+            model_name='outboundwebhook',
+            constraint=models.UniqueConstraint(fields=('ref', 'endpoint'), name='prevent_duplicate_hooks_api_outboundwebhook'),
+        ),
+    ]
@@ -0,0 +1,24 @@
+# Generated by Django 5.1 on 2024-08-20 10:44
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0003_rename_user_apitoken_created_by_apitoken_abid_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='uuid',
+            field=models.UUIDField(blank=True, editable=False, null=True, unique=True),
+        ),
+    ]
@@ -0,0 +1,22 @@
+# Generated by Django 5.1 on 2024-08-20 22:40
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0004_alter_apitoken_id_alter_apitoken_uuid'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='apitoken',
+            name='uuid',
+        ),
+        migrations.RemoveField(
+            model_name='outboundwebhook',
+            name='id',
+        ),
+    ]
@@ -0,0 +1,29 @@
+# Generated by Django 5.1 on 2024-08-20 22:43
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='outboundwebhook',
+            old_name='uuid',
+            new_name='id'
+        ),
+        migrations.AlterField(
+            model_name='outboundwebhook',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='apitoken',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False),
+        ),
+    ]
@@ -0,0 +1,22 @@
+# Generated by Django 5.1 on 2024-08-20 22:52
+
+import abid_utils.models
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0006_remove_outboundwebhook_uuid_apitoken_id_and_more'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='apitoken',
+            name='created_by',
+            field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+    ]
@@ -1,54 +1,122 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

+import uuid
 import secrets
-from archivebox.uuid_compat import CompactUUIDField, uuid7
+from datetime import timedelta

 from django.conf import settings
 from django.db import models
 from django.utils import timezone
-from django_stubs_ext.db.models import TypedModelMeta
+
 from signal_webhooks.models import WebhookBase

-from archivebox.base_models.models import get_or_create_system_user_pk
+from django_stubs_ext.db.models import TypedModelMeta
+
+from abid_utils.models import ABIDModel, ABIDField, get_or_create_system_user_pk
+


 def generate_secret_token() -> str:
+    # returns cryptographically secure string with len() == 32
    return secrets.token_hex(16)


-class APIToken(models.Model):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
+class APIToken(ABIDModel):
+    """
+    A secret key generated by a User that's used to authenticate REST API requests to ArchiveBox.
+    """
+    # ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
+    abid_prefix = 'apt_'
+    abid_ts_src = 'self.created'
+    abid_uri_src = 'self.token'
+    abid_subtype_src = 'self.created_by_id'
+    abid_rand_src = 'self.id'
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    created = models.DateTimeField(auto_now_add=True)
+
    token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
    expires = models.DateTimeField(null=True, blank=True)

    class Meta(TypedModelMeta):
-        app_label = "api"
        verbose_name = "API Key"
        verbose_name_plural = "API Keys"

    def __str__(self) -> str:
        return self.token

+    def __repr__(self) -> str:
+        return f'<APIToken user={self.created_by.username} token={self.token_redacted}>'
+
+    def __json__(self) -> dict:
+        return {
+            "TYPE":             "APIToken",    
+            "id":               str(self.pk),
+            "abid":             str(self.ABID),
+            "created_by_id":    str(self.created_by_id),
+            "token":            self.token,
+            "created":          self.created.isoformat(),
+            "expires":          self.expires_as_iso8601,
+        }
+
+    @property
+    def expires_as_iso8601(self):
+        """Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
+        expiry_date = self.expires or (timezone.now() + timedelta(days=365 * 100))
+
+        return expiry_date.isoformat()
+    
    @property
    def token_redacted(self):
-        return f"************{self.token[-4:]}"
+        return f'************{self.token[-4:]}'

    def is_valid(self, for_date=None):
-        return not self.expires or self.expires >= (for_date or timezone.now())
+        for_date = for_date or timezone.now()
+
+        if self.expires and self.expires < for_date:
+            return False
+
+        return True


-class OutboundWebhook(WebhookBase):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
+
+
+
+
+# monkey patch django-signals-webhooks to change how it shows up in Admin UI
+
+class OutboundWebhook(ABIDModel, WebhookBase):
+    """
+    Model used in place of (extending) signals_webhooks.models.WebhookModel. Swapped using:
+        settings.SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
+    """
+    abid_prefix = 'whk_'
+    abid_ts_src = 'self.created'
+    abid_uri_src = 'self.endpoint'
+    abid_subtype_src = 'self.ref'
+    abid_rand_src = 'self.id'
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    abid = ABIDField(prefix=abid_prefix)
+
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+
+    # More fields here: WebhookBase...
+
+    WebhookBase._meta.get_field('name').help_text = (
+        'Give your webhook a descriptive name (e.g. Notify ACME Slack channel of any new ArchiveResults).')
+    WebhookBase._meta.get_field('signal').help_text = (
+        'The type of event the webhook should fire for (e.g. Create, Update, Delete).')
+    WebhookBase._meta.get_field('ref').help_text = (
+        'Dot import notation of the model the webhook should fire for (e.g. core.models.Snapshot or core.models.ArchiveResult).')
+    WebhookBase._meta.get_field('endpoint').help_text = (
+        'External URL to POST the webhook notification to (e.g. https://someapp.example.com/webhook/some-webhook-receiver).')

    class Meta(WebhookBase.Meta):
-        app_label = "api"
-        verbose_name = "API Outbound Webhook"
+        verbose_name = 'API Outbound Webhook'

-    def __str__(self) -> str:
-        return f"[{self.id}] {self.ref} -> {self.endpoint}"
@@ -1,17 +0,0 @@
-"""Fields shared by responses for user-owned model objects."""
-
-from ninja import Schema
-
-
-class OwnedObjectSchema(Schema):
-    created_by_id: str
-    created_by_username: str
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by.pk)
-
-    @staticmethod
-    def resolve_created_by_username(obj) -> str:
-        user = obj.created_by
-        return user.username if isinstance(user.username, str) else str(user)
@@ -0,0 +1,30 @@
+__package__ = 'archivebox.api'
+
+from django.test import TestCase
+from ninja.testing import TestClient
+
+from .routes_cli import router
+
+class ArchiveBoxCLIAPITestCase(TestCase):
+    def setUp(self):
+        self.client = TestClient(router)
+
+    def test_add_endpoint(self):
+        response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])
+
+    def test_remove_endpoint(self):
+        response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])
+
+    def test_update_endpoint(self):
+        response = self.client.post("/update", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])
+
+    def test_list_all_endpoint(self):
+        response = self.client.post("/list_all", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.json()["success"])
@@ -1,26 +1,17 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from urllib.parse import quote
-
-from django.http import HttpRequest, HttpResponseRedirect
-from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView

-from archivebox.core.routes_util import build_web_url
 from .v1_api import urls as v1_api_urls

-
-def archive_redirect_view(request: HttpRequest, url: str) -> HttpResponseRedirect:
-    if request.META.get("QUERY_STRING"):
-        url = f"{url}?{request.META['QUERY_STRING']}"
-    return redirect(build_web_url(f"/web/{quote(url, safe=':/')}", request=request))
-
-
 urlpatterns = [
-    path("", RedirectView.as_view(url="/api/v1/docs")),
-    path("archive/<path:url>", archive_redirect_view, name="api-archive-redirect"),
-    path("v1/", RedirectView.as_view(url="/api/v1/docs")),
-    path("v1/", v1_api_urls),
-    path("v1", RedirectView.as_view(url="/api/v1/docs")),
+    path("",                 RedirectView.as_view(url='/api/v1')),
+
+    path("v1/",              v1_api_urls),
+    path("v1",               RedirectView.as_view(url='/api/v1/docs')),
+
+    # ... v2 can be added here ...
+    # path("v2/",              v2_api_urls),
+    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
 ]
@@ -1,4 +1,4 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'


 from io import StringIO
@@ -7,22 +7,18 @@ from contextlib import redirect_stdout, redirect_stderr

 from django.http import HttpRequest, HttpResponse
 from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
-from django.contrib.auth.models import User

 from ninja import NinjaAPI, Swagger

 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/

-from archivebox.config import VERSION
-from archivebox.config.version import get_COMMIT_HASH
-
-from archivebox.api.auth import API_AUTH_METHODS
-from archivebox.api.models import APIToken
+from api.auth import API_AUTH_METHODS
+from ..config import VERSION, COMMIT_HASH


-COMMIT_HASH = get_COMMIT_HASH() or "unknown"
+COMMIT_HASH = COMMIT_HASH or 'unknown'

-html_description = f"""
+html_description=f'''
 <h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
 <br/>
 <i><b>WARNING: This API is still in an early development stage and may change!</b></i>
@@ -35,59 +31,42 @@ html_description = f"""
 <li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
 </ul>
 <small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
-"""
+'''


 def register_urls(api: NinjaAPI) -> NinjaAPI:
-    api.add_router("/auth/", "archivebox.api.v1_auth.router")
-    api.add_router("/core/", "archivebox.api.v1_core.router")
-    api.add_router("/crawls/", "archivebox.api.v1_crawls.router")
-    api.add_router("/cli/", "archivebox.api.v1_cli.router")
-    api.add_router("/machine/", "archivebox.api.v1_machine.router")
-    api.add_router("/personas/", "archivebox.api.v1_personas.router")
+    api.add_router('/auth/',     'api.v1_auth.router')
+    api.add_router('/core/',     'api.v1_core.router')
+    api.add_router('/cli/',      'api.v1_cli.router')
    return api


-class NinjaAPIWithIOCapture(NinjaAPI):
+class NinjaAPIWithIOCapture(NinjaAPI):    
    def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
        stdout, stderr = StringIO(), StringIO()

        with redirect_stderr(stderr):
            with redirect_stdout(stdout):
-                setattr(request, "stdout", stdout)
-                setattr(request, "stderr", stderr)
+                request.stdout = stdout
+                request.stderr = stderr

                response = super().create_temporal_response(request)

-        # Disable caching of API responses entirely
-        response["Cache-Control"] = "no-store"
-
-        # Add debug stdout and stderr headers to response
-        response["X-ArchiveBox-Stdout"] = stdout.getvalue().replace("\n", "\\n")[:200]
-        response["X-ArchiveBox-Stderr"] = stderr.getvalue().replace("\n", "\\n")[:200]
-        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
-
-        # Add Auth Headers to response
-        api_token_attr = request.__dict__.get("_api_token")
-        api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
-        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
-
-        response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
-        response["X-ArchiveBox-Auth-Expires"] = token_expiry
-        response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
-        response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
+        print('RESPONDING NOW', response)

        return response


 api = NinjaAPIWithIOCapture(
-    title="ArchiveBox API",
+    title='ArchiveBox API',
    description=html_description,
-    version=VERSION,
+    version='1.0.0',
+    csrf=False,
    auth=API_AUTH_METHODS,
    urls_namespace="api-1",
    docs=Swagger(settings={"persistAuthorization": True}),
+    # docs_decorator=login_required,
+    # renderer=ORJSONRenderer(),
 )
 api = register_urls(api)
 urls = api.urls
@@ -99,16 +78,34 @@ def generic_exception_handler(request, err):
    if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
        status = 404

-    print("".join(format_exception(err)))
+    print(''.join(format_exception(err)))

    return api.create_response(
        request,
        {
            "succeeded": False,
-            "message": f"{err.__class__.__name__}: {err}",
+            "message": f'{err.__class__.__name__}: {err}',
            "errors": [
-                "".join(format_exception(err)),
+                ''.join(format_exception(err)),
+                # or send simpler parent-only traceback:
+                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
            ],
        },
        status=status,
    )
+
+
+
+# import orjson
+# from ninja.renderers import BaseRenderer
+# class ORJSONRenderer(BaseRenderer):
+#     media_type = "application/json"
+#     def render(self, request, data, *, response_status):
+#         return {
+#             "success": True,
+#             "errors": [],
+#             "result": data,
+#             "stdout": ansi_to_html(stdout.getvalue().strip()),
+#             "stderr": ansi_to_html(stderr.getvalue().strip()),
+#         }
+#         return orjson.dumps(data)
@@ -1,65 +1,52 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from django.http import HttpRequest
+from typing import Optional

 from ninja import Router, Schema

-from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token
+from api.models import APIToken
+from api.auth import auth_using_token, auth_using_password


-router = Router(tags=["Authentication"], auth=None)
+router = Router(tags=['Authentication'])


 class PasswordAuthSchema(Schema):
    """Schema for a /get_api_token request"""
-
-    username: str | None = None
-    password: str | None = None
+    username: Optional[str] = None
+    password: Optional[str] = None


-@router.post(
-    "/get_api_token",
-    auth=None,
-    summary="Generate an API token for a given username & password (or currently logged-in user)",
-)  # auth=None because they are not authed yet
-def get_api_token(request: HttpRequest, auth_data: PasswordAuthSchema):
+@router.post("/get_api_token", auth=None, summary='Generate an API token for a given username & password (or currently logged-in user)')             # auth=None because they are not authed yet
+def get_api_token(request, auth_data: PasswordAuthSchema):
    user = auth_using_password(
        username=auth_data.username,
        password=auth_data.password,
        request=request,
    )

-    if user and user.is_superuser:
-        api_token = get_or_create_api_token(user)
-        assert api_token is not None, "Failed to create API token"
-        return {
-            "success": True,
-            "user_id": str(user.pk),
-            "username": user.username,
-            "token": api_token.token,
-            "expires": api_token.expires.isoformat() if api_token.expires else None,
-        }
+    if user:
+        # TODO: support multiple tokens in the future, for now we just have one per user
+        api_token, created = APIToken.objects.get_or_create(created_by_id=user.pk)

+        return api_token.__json__()
+    
    return {"success": False, "errors": ["Invalid credentials"]}


+
 class TokenAuthSchema(Schema):
    """Schema for a /check_api_token request"""
-
    token: str


-@router.post(
-    "/check_api_token",
-    auth=None,
-    summary="Validate an API token to make sure its valid and non-expired",
-)  # auth=None because they are not authed yet
-def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
+@router.post("/check_api_token", auth=None, summary='Validate an API token to make sure its valid and non-expired')        # auth=None because they are not authed yet
+def check_api_token(request, token_data: TokenAuthSchema):
    user = auth_using_token(
        token=token_data.token,
        request=request,
    )
    if user:
        return {"success": True, "user_id": str(user.pk)}
-
+    
    return {"success": False, "user_id": None}
@@ -1,273 +1,234 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-import json
-from io import StringIO
-from typing import Any
+from typing import List, Dict, Any, Optional
 from enum import Enum

-from django.http import HttpRequest
-
 from ninja import Router, Schema
-from ninja.errors import HttpError
-from pydantic import Field

-from archivebox.misc.util import ansi_to_html
-from archivebox.core.models import SnapshotQuerySet
+from ..main import (
+    add,
+    remove,
+    update,
+    list_all,
+    schedule,
+)
+from ..util import ansi_to_html
+from ..config import ONLY_NEW


 # router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=["ArchiveBox CLI Sub-Commands"])
+router = Router(tags=['ArchiveBox CLI Sub-Commands'])


 # Schemas

-JSONType = list[Any] | dict[str, Any] | bool | int | str | None
-FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
-
+JSONType = List[Any] | Dict[str, Any] | bool | int | str | None

 class CLICommandResponseSchema(Schema):
    success: bool
-    errors: list[str]
+    errors: List[str]
    result: JSONType
-    result_format: str = "str"
    stdout: str
    stderr: str

+class FilterTypeChoices(str, Enum):
+    exact = 'exact'
+    substring = 'substring'
+    regex = 'regex'
+    domain = 'domain'
+    tag = 'tag'
+    timestamp = 'timestamp'

-FilterTypeChoices = Enum(
-    "FilterTypeChoices",
-    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
-    type=str,
-)
+class StatusChoices(str, Enum):
+    indexed = 'indexed'
+    archived = 'archived'
+    unarchived = 'unarchived'
+    present = 'present'
+    valid = 'valid'
+    invalid = 'invalid'
+    duplicate = 'duplicate'
+    orphaned = 'orphaned'
+    corrupted = 'corrupted'
+    unrecognized = 'unrecognized'


 class AddCommandSchema(Schema):
-    urls: list[str]
-    snapshot_ids: list[str] | None = None
+    urls: List[str]
    tag: str = ""
    depth: int = 0
-    max_urls: int = 0
-    crawl_max_size: int = 0
-    crawl_timeout: int = 0
-    snapshot_max_size: int = 0
-    parser: str = "auto"
-    plugins: str = ""
-    persona: str = "Default"
-    only_new: bool | None = None
-    update: bool = False
+    update: bool = not ONLY_NEW  # Default to the opposite of ONLY_NEW
+    update_all: bool = False
+    index_only: bool = False
    overwrite: bool = False
+    init: bool = False
+    extractors: str = ""
+    parser: str = "auto"
+
+class UpdateCommandSchema(Schema):
+    resume: Optional[float] = 0
+    only_new: bool = ONLY_NEW
    index_only: bool = False
-
-
-class SnapshotFilterCommandSchema(Schema):
-    after: float | None = 0
-    before: float | None = None
-    filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
-    status: str | None = None
-    url__icontains: str | None = None
-    url__istartswith: str | None = None
-    tag: str | None = None
-    crawl_id: str | None = None
-    limit: int | None = None
-    sort: str | None = None
-    search: str | None = None
-
-
-class UpdateCommandSchema(SnapshotFilterCommandSchema):
-    resume: str | None = None
-    batch_size: int = 100
-    continuous: bool = False
-    index_only: bool = False
-    migrate_only: bool = False
-
+    overwrite: bool = False
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
+    status: Optional[StatusChoices] = StatusChoices.unarchived
+    filter_type: Optional[str] = FilterTypeChoices.substring
+    filter_patterns: Optional[List[str]] = ['https://example.com']
+    extractors: Optional[str] = ""

 class ScheduleCommandSchema(Schema):
-    import_path: str | None = None
+    import_path: Optional[str] = None
    add: bool = False
-    show: bool = False
-    foreground: bool = False
-    run_all: bool = False
-    quiet: bool = False
-    every: str | None = None
-    tag: str = ""
+    every: Optional[str] = None
+    tag: str = ''
    depth: int = 0
-    only_new: bool | None = None
-    update: bool = False
    overwrite: bool = False
+    update: bool = not ONLY_NEW
    clear: bool = False

-
-class ListCommandSchema(SnapshotFilterCommandSchema):
+class ListCommandSchema(Schema):
+    filter_patterns: Optional[List[str]] = ['https://example.com']
+    filter_type: str = FilterTypeChoices.substring
+    status: Optional[StatusChoices] = StatusChoices.indexed
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
+    sort: str = 'added'
    as_json: bool = True
    as_html: bool = False
-    as_csv: str | None = "timestamp,url"
+    as_csv: str | bool = 'timestamp,url'
    with_headers: bool = False

-
-class RemoveCommandSchema(SnapshotFilterCommandSchema):
+class RemoveCommandSchema(Schema):
+    delete: bool = True
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
    filter_type: str = FilterTypeChoices.exact
-    timeout: float = 60.0
+    filter_patterns: Optional[List[str]] = ['https://example.com']


-def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
-    kwargs = args.dict()
-    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
-    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
-    return kwargs


-def cli_response(request, result, *, result_format="json", success=True, errors=()):
-    """Use one response envelope for CLI-backed endpoints, including captured output."""
-    payload = {"success": success, "errors": list(errors), "result": result, "result_format": result_format}
-    for stream_name in ("stdout", "stderr"):
-        stream = request.__dict__.get(stream_name)
-        payload[stream_name] = ansi_to_html(stream.getvalue().strip()) if isinstance(stream, StringIO) else ""
-    return payload

-
-def command_config(args) -> dict[str, object] | None:
-    """Translate shared legacy add/schedule flags into runtime config overrides."""
-    if args.update or args.overwrite:
-        return {"ONLY_NEW": False}
-    return {"ONLY_NEW": bool(args.only_new)} if args.only_new is not None else None
-
-
-@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
-def cli_add(request: HttpRequest, args: AddCommandSchema):
-    from archivebox.cli.archivebox_add import add
-    from archivebox.misc.util import validate_url
-
-    submitted_urls: str | list[str] = args.urls
-    if len(args.urls) == 1:
-        try:
-            validate_url(args.urls[0])
-        except ValueError:
-            submitted_urls = args.urls[0]
-    else:
-        submitted_urls = []
-        for url in args.urls:
-            try:
-                submitted_urls.append(validate_url(url))
-            except ValueError:
-                continue
-        if not submitted_urls:
-            raise HttpError(400, "No valid URLs were submitted")
-    crawl, snapshots = add(
-        urls=submitted_urls,
-        **args.model_dump(exclude={"urls", "only_new", "update", "overwrite"}),
-        bg=True,  # Always run in background for API calls
-        created_by_id=request.user.pk,
-        config=command_config(args),
+@router.post("/add", response=CLICommandResponseSchema, summary='archivebox add [args] [urls]')
+def cli_add(request, args: AddCommandSchema):
+    result = add(
+        urls=args.urls,
+        tag=args.tag,
+        depth=args.depth,
+        update=args.update,
+        update_all=args.update_all,
+        index_only=args.index_only,
+        overwrite=args.overwrite,
+        init=args.init,
+        extractors=args.extractors,
+        parser=args.parser,
    )

-    snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
-    result_payload = {
-        "crawl_id": str(crawl.id),
-        "num_snapshots": len(snapshot_ids),
-        "snapshot_ids": snapshot_ids,
-        "queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
    }
-    return cli_response(request, result_payload, result_format="json")


-@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
-def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
-    from archivebox.core.snapshot_status import normalize_snapshot_status
-
-    try:
-        status = normalize_snapshot_status(args.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
-    update_kwargs["status"] = status
-    update_kwargs["stop_daemon_stack"] = False
-
-    is_filtered_update = any(
-        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
+@router.post("/update", response=CLICommandResponseSchema, summary='archivebox update [args] [filter_patterns]')
+def cli_update(request, args: UpdateCommandSchema):
+    result = update(
+        resume=args.resume,
+        only_new=args.only_new,
+        index_only=args.index_only,
+        overwrite=args.overwrite,
+        before=args.before,
+        after=args.after,
+        status=args.status,
+        filter_type=args.filter_type,
+        filter_patterns=args.filter_patterns,
+        extractors=args.extractors,
    )
-    matched_snapshot_ids = []
-    if is_filtered_update:
-        matched_snapshot_ids = [
-            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
-        ]
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }

-    update(**update_kwargs)
-    return cli_response(
-        request,
-        {"matched_count": len(matched_snapshot_ids), "snapshot_ids": matched_snapshot_ids} if is_filtered_update else None,
-        result_format="str",
+
+@router.post("/schedule", response=CLICommandResponseSchema, summary='archivebox schedule [args] [import_path]')
+def cli_schedule(request, args: ScheduleCommandSchema):
+    result = schedule(
+        import_path=args.import_path,
+        add=args.add,
+        show=args.show,
+        clear=args.clear,
+        every=args.every,
+        tag=args.tag,
+        depth=args.depth,
+        overwrite=args.overwrite,
+        update=args.update,
    )

-
-@router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
-def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
-    from archivebox.cli.archivebox_schedule import schedule
-
-    result = schedule(**args.model_dump(exclude={"only_new", "update", "overwrite"}), config=command_config(args))
-
-    return cli_response(request, result, result_format="json")
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }


-@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
-def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset

-    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
-    as_json = search_kwargs.pop("as_json")
-    as_html = search_kwargs.pop("as_html")
-    as_csv = search_kwargs.pop("as_csv")
-    with_headers = search_kwargs.pop("with_headers")
-    try:
-        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
+@router.post("/list", response=CLICommandResponseSchema, summary='archivebox list [args] [filter_patterns]')
+def cli_list(request, args: ListCommandSchema):
+    result = list_all(
+        filter_patterns=args.filter_patterns,
+        filter_type=args.filter_type,
+        status=args.status,
+        after=args.after,
+        before=args.before,
+        sort=args.sort,
+        csv=args.as_csv,
+        json=args.as_json,
+        html=args.as_html,
+        with_headers=args.with_headers,
+    )

-    result_format = "txt"
-    if as_json:
+    result_format = 'txt'
+    if args.as_json:
        result_format = "json"
-        result = [
-            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
-            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
-        ]
-    elif as_html:
+    elif args.as_html:
        result_format = "html"
-        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
-    elif as_csv:
+    elif args.as_csv:
        result_format = "csv"
-        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
-        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
-        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
-    else:
-        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))

-    return cli_response(request, result, result_format=result_format)
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "result_format": result_format,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }
+    


-@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
-def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
-    from archivebox.cli.archivebox_remove import remove
-    from archivebox.core.models import Snapshot
-
-    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
-    timeout_arg = remove_kwargs.pop("timeout")
-    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
-    try:
-        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
+@router.post("/remove", response=CLICommandResponseSchema, summary='archivebox remove [args] [filter_patterns]')
+def cli_remove(request, args: RemoveCommandSchema):
    result = remove(
-        yes=True,  # no way to interactively ask for confirmation via API, so we force yes
-        snapshots=snapshots_to_remove,
-        timeout=timeout,
-    )
-    return cli_response(
-        request,
-        result,
-        result_format="json",
-        success=bool(result["success"]),
-        errors=[str(result["error"])] if result["error"] else [],
+        yes=True,            # no way to interactively ask for confirmation via API, so we force yes
+        delete=args.delete,
+        before=args.before,
+        after=args.after,
+        filter_type=args.filter_type,
+        filter_patterns=args.filter_patterns,
    )
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }
+    
@@ -0,0 +1,424 @@
+__package__ = 'archivebox.api'
+
+import math
+from uuid import UUID
+from typing import List, Optional, Union, Any
+from datetime import datetime
+
+from django.db.models import Q
+from django.shortcuts import get_object_or_404
+from django.core.exceptions import ValidationError
+from django.contrib.auth import get_user_model
+
+from ninja import Router, Schema, FilterSchema, Field, Query
+from ninja.pagination import paginate, PaginationBase
+
+from core.models import Snapshot, ArchiveResult, Tag
+from abid_utils.abid import ABID
+
+router = Router(tags=['Core Models'])
+
+
+
+class CustomPagination(PaginationBase):
+    class Input(Schema):
+        limit: int = 200
+        offset: int = 0
+        page: int = 0
+
+
+    class Output(Schema):
+        total_items: int
+        total_pages: int
+        page: int
+        limit: int
+        offset: int
+        num_items: int
+        items: List[Any]
+
+    def paginate_queryset(self, queryset, pagination: Input, **params):
+        limit = min(pagination.limit, 500)
+        offset = pagination.offset or (pagination.page * limit)
+        total = queryset.count()
+        total_pages = math.ceil(total / limit)
+        current_page = math.ceil(offset / (limit + 1))
+        items = queryset[offset : offset + limit]
+        return {
+            'total_items': total,
+            'total_pages': total_pages,
+            'page': current_page,
+            'limit': limit,
+            'offset': offset,
+            'num_items': len(items),
+            'items': items,
+        }
+
+
+### ArchiveResult #########################################################################
+
+class ArchiveResultSchema(Schema):
+    TYPE: str = 'core.models.ArchiveResult'
+
+    id: UUID
+    old_id: int
+    abid: str
+
+    modified: datetime
+    created: datetime
+    created_by_id: str
+    created_by_username: str
+
+    snapshot_abid: str
+    snapshot_timestamp: str
+    snapshot_url: str
+    snapshot_tags: str
+
+    extractor: str
+    cmd_version: Optional[str]
+    cmd: List[str]
+    pwd: str
+    status: str
+    output: str
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+    
+    @staticmethod
+    def resolve_created_by_username(obj):
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username
+
+    @staticmethod
+    def resolve_pk(obj):
+        return str(obj.pk)
+
+    @staticmethod
+    def resolve_uuid(obj):
+        return str(obj.uuid)
+
+    @staticmethod
+    def resolve_abid(obj):
+        return str(obj.ABID)
+
+    @staticmethod
+    def resolve_created(obj):
+        return obj.start_ts
+
+    @staticmethod
+    def resolve_snapshot_timestamp(obj):
+        return obj.snapshot.timestamp
+    
+    @staticmethod
+    def resolve_snapshot_url(obj):
+        return obj.snapshot.url
+
+    @staticmethod
+    def resolve_snapshot_abid(obj):
+        return str(obj.snapshot.ABID)
+
+    @staticmethod
+    def resolve_snapshot_tags(obj):
+        return obj.snapshot.tags_str()
+
+
+class ArchiveResultFilterSchema(FilterSchema):
+    id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+
+    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+    snapshot_id: Optional[str] = Field(None, q=['snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+    snapshot_url: Optional[str] = Field(None, q='snapshot__url__icontains')
+    snapshot_tag: Optional[str] = Field(None, q='snapshot__tags__name__icontains')
+    
+    status: Optional[str] = Field(None, q='status')
+    output: Optional[str] = Field(None, q='output__icontains')
+    extractor: Optional[str] = Field(None, q='extractor__icontains')
+    cmd: Optional[str] = Field(None, q='cmd__0__icontains')
+    pwd: Optional[str] = Field(None, q='pwd__icontains')
+    cmd_version: Optional[str] = Field(None, q='cmd_version')
+
+    created: Optional[datetime] = Field(None, q='updated')
+    created__gte: Optional[datetime] = Field(None, q='updated__gte')
+    created__lt: Optional[datetime] = Field(None, q='updated__lt')
+
+
+@router.get("/archiveresults", response=List[ArchiveResultSchema], url_name="get_archiveresult")
+@paginate(CustomPagination)
+def get_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...)):
+    """List all ArchiveResult entries matching these filters."""
+    qs = ArchiveResult.objects.all()
+    results = filters.filter(qs).distinct()
+    return results
+
+
+@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
+def get_archiveresult(request, archiveresult_id: str):
+    """Get a specific ArchiveResult by pk, abid, or old_id."""
+    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id) | Q(old_id__icontains=archiveresult_id))
+
+
+# @router.post("/archiveresult", response=ArchiveResultSchema)
+# def create_archiveresult(request, payload: ArchiveResultSchema):
+#     archiveresult = ArchiveResult.objects.create(**payload.dict())
+#     return archiveresult
+#
+# @router.put("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema)
+# def update_archiveresult(request, archiveresult_id: str, payload: ArchiveResultSchema):
+#     archiveresult = get_object_or_404(ArchiveResult, id=archiveresult_id)
+#   
+#     for attr, value in payload.dict().items():
+#         setattr(archiveresult, attr, value)
+#     archiveresult.save()
+#
+#     return archiveresult
+#
+# @router.delete("/archiveresult/{archiveresult_id}")
+# def delete_archiveresult(request, archiveresult_id: str):
+#     archiveresult = get_object_or_404(ArchiveResult, id=archiveresult_id)
+#     archiveresult.delete()
+#     return {"success": True}
+
+
+
+
+
+### Snapshot #########################################################################
+
+
+class SnapshotSchema(Schema):
+    TYPE: str = 'core.models.Snapshot'
+
+    id: UUID
+    old_id: UUID
+    abid: str
+
+    modified: datetime
+    created: datetime
+    created_by_id: str
+    created_by_username: str
+
+    url: str
+    tags: str
+    title: Optional[str]
+    timestamp: str
+    archive_path: str
+
+    bookmarked: datetime
+    added: datetime
+    updated: Optional[datetime]
+
+    num_archiveresults: int
+    archiveresults: List[ArchiveResultSchema]
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+    
+    @staticmethod
+    def resolve_created_by_username(obj):
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username
+
+    @staticmethod
+    def resolve_pk(obj):
+        return str(obj.pk)
+
+    @staticmethod
+    def resolve_uuid(obj):
+        return str(obj.uuid)
+
+    @staticmethod
+    def resolve_abid(obj):
+        return str(obj.ABID)
+
+    @staticmethod
+    def resolve_tags(obj):
+        return obj.tags_str()
+
+    @staticmethod
+    def resolve_num_archiveresults(obj, context):
+        return obj.archiveresult_set.all().distinct().count()
+
+    @staticmethod
+    def resolve_archiveresults(obj, context):
+        if context['request'].with_archiveresults:
+            return obj.archiveresult_set.all().distinct()
+        return ArchiveResult.objects.none()
+
+
+class SnapshotFilterSchema(FilterSchema):
+    id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
+
+    old_id: Optional[str] = Field(None, q='old_id__icontains')
+    abid: Optional[str] = Field(None, q='abid__icontains')
+
+    created_by_id: str = Field(None, q='created_by_id')
+    created_by_username: str = Field(None, q='created_by__username__icontains')
+
+    created__gte: datetime = Field(None, q='created__gte')
+    created__lt: datetime = Field(None, q='created__lt')
+    created: datetime = Field(None, q='created')
+    modified: datetime = Field(None, q='modified')
+    modified__gte: datetime = Field(None, q='modified__gte')
+    modified__lt: datetime = Field(None, q='modified__lt')
+
+    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
+    url: Optional[str] = Field(None, q='url')
+    tag: Optional[str] = Field(None, q='tags__name')
+    title: Optional[str] = Field(None, q='title__icontains')
+    timestamp: Optional[str] = Field(None, q='timestamp__startswith')
+    
+    added__gte: Optional[datetime] = Field(None, q='added__gte')
+    added__lt: Optional[datetime] = Field(None, q='added__lt')
+
+
+
+@router.get("/snapshots", response=List[SnapshotSchema], url_name="get_snapshots")
+@paginate(CustomPagination)
+def get_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_archiveresults: bool=False):
+    """List all Snapshot entries matching these filters."""
+    request.with_archiveresults = with_archiveresults
+
+    qs = Snapshot.objects.all()
+    results = filters.filter(qs).distinct()
+    return results
+
+@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
+def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
+    """Get a specific Snapshot by abid, uuid, or pk."""
+    request.with_archiveresults = with_archiveresults
+    snapshot = None
+    try:
+        snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(old_id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
+    except Snapshot.DoesNotExist:
+        pass
+
+    try:
+        snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
+    except Snapshot.DoesNotExist:
+        pass
+
+    if not snapshot:
+        raise Snapshot.DoesNotExist
+
+    return snapshot
+
+
+# @router.post("/snapshot", response=SnapshotSchema)
+# def create_snapshot(request, payload: SnapshotSchema):
+#     snapshot = Snapshot.objects.create(**payload.dict())
+#     return snapshot
+#
+# @router.put("/snapshot/{snapshot_id}", response=SnapshotSchema)
+# def update_snapshot(request, snapshot_id: str, payload: SnapshotSchema):
+#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
+#
+#     for attr, value in payload.dict().items():
+#         setattr(snapshot, attr, value)
+#     snapshot.save()
+#
+#     return snapshot
+#
+# @router.delete("/snapshot/{snapshot_id}")
+# def delete_snapshot(request, snapshot_id: str):
+#     snapshot = get_object_or_404(Snapshot, uuid=snapshot_id)
+#     snapshot.delete()
+#     return {"success": True}
+
+
+
+### Tag #########################################################################
+
+
+class TagSchema(Schema):
+    TYPE: str = 'core.models.Tag'
+
+    id: UUID
+    old_id: str
+    abid: str
+
+    modified: datetime
+    created: datetime
+    created_by_id: str
+    created_by_username: str
+
+    name: str
+    slug: str
+    num_snapshots: int
+    snapshots: List[SnapshotSchema]
+
+    @staticmethod
+    def resolve_old_id(obj):
+        return str(obj.old_id)
+
+    @staticmethod
+    def resolve_created_by_id(obj):
+        return str(obj.created_by_id)
+    
+    @staticmethod
+    def resolve_created_by_username(obj):
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username
+    
+    @staticmethod
+    def resolve_num_snapshots(obj, context):
+        return obj.snapshot_set.all().distinct().count()
+
+    @staticmethod
+    def resolve_snapshots(obj, context):
+        if context['request'].with_snapshots:
+            return obj.snapshot_set.all().distinct()
+        return Snapshot.objects.none()
+
+@router.get("/tags", response=List[TagSchema], url_name="get_tags")
+@paginate(CustomPagination)
+def get_tags(request):
+    request.with_snapshots = False
+    request.with_archiveresults = False
+    return Tag.objects.all().distinct()
+
+@router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
+def get_tag(request, tag_id: str, with_snapshots: bool=True):
+    request.with_snapshots = with_snapshots
+    request.with_archiveresults = False
+    tag = None
+    try:
+        tag = tag or Tag.objects.get(old_id__icontains=tag_id)
+    except (Tag.DoesNotExist, ValidationError, ValueError):
+        pass
+
+    try:
+        tag = Tag.objects.get(abid__icontains=tag_id)
+    except (Tag.DoesNotExist, ValidationError):
+        pass
+
+    try:
+        tag = tag or Tag.objects.get(id__icontains=tag_id)
+    except (Tag.DoesNotExist, ValidationError):
+        pass
+    return tag
+
+
+
+@router.get("/any/{abid}", response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema], url_name="get_any")
+def get_any(request, abid: str):
+    request.with_snapshots = False
+    request.with_archiveresults = False
+
+    response = None
+    try:
+        response = response or get_snapshot(request, abid)
+    except Exception:
+        pass
+
+    try:
+        response = response or get_archiveresult(request, abid)
+    except Exception:
+        pass
+
+    try:
+        response = response or get_tag(request, abid)
+    except Exception:
+        pass
+
+    return response
@@ -1,47 +0,0 @@
-"""Core REST routes, composed from entity-specific routers."""
-
-from typing import Union
-
-from django.db.models import Model
-from django.http import HttpRequest
-from django.shortcuts import redirect
-from ninja import Router
-from ninja.errors import HttpError
-
-from archivebox.api.v1_crawls import CrawlSchema, get_crawl
-
-from . import archiveresults, snapshots, tags
-from .archiveresults import get_archiveresult
-from .pagination import CustomPagination as CustomPagination
-from .schemas import ArchiveResultSchema, SnapshotSchema, TagSchema
-from .snapshots import get_snapshot
-from .tags import get_tag
-
-router = Router(tags=["Core Models"])
-router.add_router("", archiveresults.router)
-router.add_router("", snapshots.router)
-router.add_router("", tags.router)
-
-
-@router.get(
-    "/any/{id}",
-    response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema, CrawlSchema],
-    url_name="get_any",
-    summary="Get any object by its ID",
-)
-def get_any(request: HttpRequest, id: str):
-    """Get any object by its ID (e.g. snapshot, archiveresult, tag, crawl, etc.)."""
-    setattr(request, "with_snapshots", False)
-    setattr(request, "with_archiveresults", False)
-
-    for getter in (get_snapshot, get_archiveresult, get_tag, get_crawl):
-        try:
-            response = getter(request, id)
-            if isinstance(response, Model):
-                return redirect(
-                    f"/api/v1/{response._meta.app_label}/{response._meta.model_name}/{response.pk}?{request.META['QUERY_STRING']}",
-                )
-        except Exception:
-            pass
-
-    raise HttpError(404, "Object with given ID not found")
@@ -1,310 +0,0 @@
-import json
-import mimetypes
-import re
-from pathlib import Path, PurePosixPath
-from typing import Any
-
-from django.core.files.storage import FileSystemStorage
-from django.http import HttpRequest
-from django.http.multipartparser import MultiPartParser, MultiPartParserError
-from ninja import Form, Query, Router, UploadedFile
-from ninja.errors import HttpError
-from ninja.pagination import paginate
-
-from archivebox.core.models import ArchiveResult, Snapshot
-from archivebox.misc.db import uuid_ref_query as _uuid_ref_query
-
-from .lookup import _get_snapshot_by_ref
-from .pagination import CustomPagination
-from .schemas import ArchiveResultFilterSchema, ArchiveResultSchema
-
-router = Router(tags=["Core Models"])
-
-ARCHIVERESULT_UPLOAD_HOOK_NAME = Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME
-ARCHIVERESULT_UPLOAD_PLUGIN_RE = re.compile(r"^[A-Za-z0-9_.-]{1,32}$")
-
-
-@router.get("/archiveresults", response=list[ArchiveResultSchema], url_name="get_archiveresult")
-@paginate(CustomPagination)
-def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
-    """List all ArchiveResult entries matching these filters."""
-    queryset = filters.filter(ArchiveResult.objects.all())
-    if filters.search or filters.snapshot_tag:
-        return queryset.distinct()
-    return queryset
-
-
-@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
-def get_archiveresult(request: HttpRequest, archiveresult_id: str):
-    """Get a specific ArchiveResult by id."""
-    return ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
-
-
-def _normalize_uploaded_archiveresult_plugin(plugin: str) -> str:
-    normalized = str(plugin or "").strip().strip("/")
-    if not ARCHIVERESULT_UPLOAD_PLUGIN_RE.fullmatch(normalized):
-        raise HttpError(400, "Invalid ArchiveResult plugin name")
-    return normalized
-
-
-def _normalize_uploaded_archiveresult_output_path(output_path: str, *, filename: str) -> str:
-    raw_path = str(output_path or filename or "").strip().replace("\\", "/")
-    if not raw_path:
-        raise HttpError(400, "ArchiveResult output path is required")
-
-    path = PurePosixPath(raw_path)
-    if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts):
-        raise HttpError(400, "Invalid ArchiveResult output path")
-
-    return str(path)
-
-
-def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any] | None:
-    if not output_json:
-        return None
-    try:
-        parsed = json.loads(output_json)
-    except json.JSONDecodeError as err:
-        raise HttpError(400, "ArchiveResult output_json must be valid JSON") from err
-    if parsed is None:
-        return None
-    if not isinstance(parsed, dict):
-        raise HttpError(400, "ArchiveResult output_json must be a JSON object")
-    return parsed
-
-
-def _get_archiveresult_upload_data(request: HttpRequest):
-    cached = request.__dict__.get("_archiveresult_upload_data")
-    if cached is not None:
-        return cached
-
-    if request.method.upper() == "PATCH" and request.content_type.startswith("multipart/"):
-        try:
-            data = MultiPartParser(request.META, request, request.upload_handlers, request.encoding).parse()
-        except MultiPartParserError as err:
-            raise HttpError(400, f"Invalid ArchiveResult multipart upload: {err}") from err
-    else:
-        data = (request.POST, request.FILES)
-
-    setattr(request, "_archiveresult_upload_data", data)
-    return data
-
-
-def _get_archiveresult_upload_files(request: HttpRequest, *, allow_empty: bool = False) -> list[UploadedFile]:
-    _post, request_files = _get_archiveresult_upload_data(request)
-    files = [*request_files.getlist("files"), *request_files.getlist("file")]
-    if not files and not allow_empty:
-        raise HttpError(400, "At least one ArchiveResult file is required")
-    return files
-
-
-def _get_archiveresult_upload_form_values(request: HttpRequest, *field_names: str) -> list[str]:
-    request_post, _files = _get_archiveresult_upload_data(request)
-    values: list[str] = []
-    for field_name in field_names:
-        values.extend(str(value) for value in request_post.getlist(field_name))
-    if len(values) == 1:
-        value = values[0].strip()
-        if value.startswith("["):
-            try:
-                parsed = json.loads(value)
-            except json.JSONDecodeError:
-                parsed = None
-            if isinstance(parsed, list):
-                return [str(item) for item in parsed]
-    return values
-
-
-def _get_archiveresult_upload_form_value(request: HttpRequest, *field_names: str) -> str:
-    request_post, _files = _get_archiveresult_upload_data(request)
-    for field_name in field_names:
-        value = request_post.get(field_name)
-        if value is not None:
-            return str(value)
-    return ""
-
-
-def _parse_archiveresult_upload_int(value: str, field_name: str, *, default: int | None = None) -> int:
-    if value == "" and default is not None:
-        return default
-    try:
-        parsed = int(value)
-    except (TypeError, ValueError) as err:
-        raise HttpError(400, f"ArchiveResult {field_name} must be an integer") from err
-    if parsed < 0:
-        raise HttpError(400, f"ArchiveResult {field_name} must be non-negative")
-    return parsed
-
-
-def _write_archiveresult_files(
-    request: HttpRequest,
-    snapshot: Snapshot,
-    plugin_name: str,
-    *,
-    existing_output_files: dict[str, dict[str, Any]] | None = None,
-    allow_empty: bool = False,
-) -> dict[str, dict[str, Any]]:
-    files = _get_archiveresult_upload_files(request, allow_empty=allow_empty)
-    output_paths = _get_archiveresult_upload_form_values(request, "output_paths", "output_path")
-    mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
-    chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
-
-    snapshot_dir = snapshot.output_dir
-    plugin_dir = snapshot_dir / plugin_name
-    storage = FileSystemStorage(location=str(plugin_dir))
-    output_files = dict(existing_output_files or {})
-
-    if not files:
-        return output_files
-
-    if chunk_output_path:
-        if len(files) != 1:
-            raise HttpError(400, "Exactly one ArchiveResult file chunk is required")
-
-        uploaded_file = files[0]
-        relative_output_path = _normalize_uploaded_archiveresult_output_path(
-            chunk_output_path,
-            filename=uploaded_file.name,
-        )
-        chunk_index = _parse_archiveresult_upload_int(
-            _get_archiveresult_upload_form_value(request, "chunk_index"),
-            "chunk_index",
-        )
-        chunk_count = _parse_archiveresult_upload_int(
-            _get_archiveresult_upload_form_value(request, "chunk_count"),
-            "chunk_count",
-        )
-        chunk_offset = _parse_archiveresult_upload_int(
-            _get_archiveresult_upload_form_value(request, "chunk_offset"),
-            "chunk_offset",
-        )
-        chunk_total_size = _parse_archiveresult_upload_int(
-            _get_archiveresult_upload_form_value(request, "chunk_total_size"),
-            "chunk_total_size",
-        )
-
-        if chunk_count < 1:
-            raise HttpError(400, "ArchiveResult chunk_count must be at least 1")
-        if chunk_index >= chunk_count:
-            raise HttpError(400, "ArchiveResult chunk_index must be less than chunk_count")
-        if chunk_total_size and chunk_offset > chunk_total_size:
-            raise HttpError(400, "ArchiveResult chunk_offset cannot exceed chunk_total_size")
-
-        if chunk_index == 0 and chunk_offset == 0 and storage.exists(relative_output_path):
-            storage.delete(relative_output_path)
-
-        current_size = storage.size(relative_output_path) if storage.exists(relative_output_path) else 0
-        if current_size != chunk_offset:
-            raise HttpError(
-                409,
-                f"ArchiveResult chunk offset mismatch for {relative_output_path}: expected {current_size}, got {chunk_offset}",
-            )
-
-        Path(storage.path(relative_output_path)).parent.mkdir(parents=True, exist_ok=True)
-        with storage.open(relative_output_path, "ab") as destination:
-            for chunk in uploaded_file.chunks():
-                destination.write(chunk)
-
-        size = storage.size(relative_output_path)
-        upload_complete = chunk_index + 1 == chunk_count
-        if upload_complete and size != chunk_total_size:
-            raise HttpError(
-                409,
-                f"ArchiveResult chunk size mismatch for {relative_output_path}: expected {chunk_total_size}, got {size}",
-            )
-
-        guessed_mime = mimetypes.guess_type(relative_output_path)[0]
-        output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
-        output_files[relative_output_path] = {
-            "extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
-            "mimetype": output_mime_type,
-            "size": size,
-            "upload": {
-                "chunked": True,
-                "chunk_index": chunk_index,
-                "chunk_count": chunk_count,
-                "chunks_received": chunk_index + 1,
-                "complete": upload_complete,
-            },
-        }
-
-        return output_files
-
-    for index, uploaded_file in enumerate(files):
-        relative_output_path = _normalize_uploaded_archiveresult_output_path(
-            output_paths[index] if index < len(output_paths) else "",
-            filename=uploaded_file.name,
-        )
-        if storage.exists(relative_output_path):
-            storage.delete(relative_output_path)
-        saved_output_path = storage.save(relative_output_path, uploaded_file)
-        size = storage.size(saved_output_path)
-        guessed_mime = mimetypes.guess_type(saved_output_path)[0]
-        output_mime_type = (
-            (mime_types[index] if index < len(mime_types) else "")
-            or uploaded_file.content_type
-            or guessed_mime
-            or "application/octet-stream"
-        )
-        output_files[saved_output_path] = {
-            "extension": PurePosixPath(saved_output_path).suffix.lower().lstrip("."),
-            "mimetype": output_mime_type,
-            "size": size,
-        }
-
-    return output_files
-
-
-@router.post(
-    "/archiveresults",
-    response=ArchiveResultSchema,
-    url_name="create_archiveresult",
-)
-def create_archiveresult(
-    request: HttpRequest,
-    snapshot_id: str = Form(...),
-    plugin: str = Form(...),
-    output_str: str = Form(""),
-    hook_name: str = Form(ARCHIVERESULT_UPLOAD_HOOK_NAME),
-    status: str = Form(str(ArchiveResult.StatusChoices.SUCCEEDED)),
-    output_json: str = Form(""),
-):
-    """Create or update an ArchiveResult with one or more output files."""
-    snapshot = _get_snapshot_by_ref(snapshot_id)
-    plugin_name = _normalize_uploaded_archiveresult_plugin(plugin)
-    result = ArchiveResult.objects.filter(
-        snapshot=snapshot,
-        plugin=plugin_name,
-        hook_name=hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME,
-    ).first()
-    result = result or ArchiveResult(snapshot=snapshot, plugin=plugin_name, hook_name=hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME)
-    metadata = {"output_str": output_str, "status": status, "output_json": _parse_archiveresult_output_json(output_json)}
-    files = _write_archiveresult_files(request, snapshot, plugin_name, allow_empty=True)
-    try:
-        return result.apply_upload(files, metadata=metadata, replace_metadata=True)
-    except ArchiveResult.UploadConflict as err:
-        raise HttpError(409, str(err)) from err
-
-
-@router.patch("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="patch_archiveresult")
-def patch_archiveresult(
-    request: HttpRequest,
-    archiveresult_id: str,
-):
-    """Append or replace files on an existing ArchiveResult."""
-    result = ArchiveResult.objects.select_related("snapshot__crawl__created_by").get(_uuid_ref_query("id", archiveresult_id))
-    uploaded_output_files = _write_archiveresult_files(
-        request,
-        result.snapshot,
-        result.plugin,
-    )
-    output_str = _get_archiveresult_upload_form_value(request, "output_str")
-    status = _get_archiveresult_upload_form_value(request, "status")
-    output_json = _get_archiveresult_upload_form_value(request, "output_json")
-    metadata = {key: value for key, value in {"output_str": output_str, "status": status}.items() if value}
-    if output_json:
-        metadata["output_json"] = _parse_archiveresult_output_json(output_json)
-    try:
-        return result.apply_upload(uploaded_output_files, metadata=metadata)
-    except ArchiveResult.UploadConflict as err:
-        raise HttpError(409, str(err)) from err
@@ -1,12 +0,0 @@
-from django.db.models import Q
-
-from archivebox.core.models import Snapshot
-from archivebox.misc.db import uuid_ref_query as _uuid_ref_query
-
-
-def _get_snapshot_by_ref(snapshot_id: str):
-    queryset = Snapshot.objects.select_related("crawl__created_by")
-    try:
-        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        return queryset.get(_uuid_ref_query("id", snapshot_id))
@@ -1,40 +0,0 @@
-import math
-from typing import Any
-
-from django.http import HttpRequest
-from ninja.pagination import PaginationBase
-
-
-class CustomPagination(PaginationBase):
-    class Input(PaginationBase.Input):
-        limit: int = 200
-        offset: int = 0
-        page: int = 0
-
-    class Output(PaginationBase.Output):
-        count: int
-        total_items: int
-        total_pages: int
-        page: int
-        limit: int
-        offset: int
-        num_items: int
-        items: list[Any]
-
-    def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
-        limit = min(pagination.limit, 500)
-        offset = pagination.offset or (pagination.page * limit)
-        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
-        total_pages = math.ceil(total / limit)
-        current_page = math.ceil(offset / (limit + 1))
-        items = queryset[offset : offset + limit]
-        return {
-            "count": total,
-            "total_items": total,
-            "total_pages": total_pages,
-            "page": current_page,
-            "limit": limit,
-            "offset": offset,
-            "num_items": len(items),
-            "items": items,
-        }
@@ -1,301 +0,0 @@
-from collections import defaultdict
-from datetime import datetime
-from typing import Annotated, Any
-from uuid import UUID
-
-from django.db.models import Q
-from archivebox.api.schemas import OwnedObjectSchema
-
-from ninja import FilterLookup, FilterSchema, Schema
-
-from archivebox.core.models import ArchiveResult, Snapshot
-
-
-class MinimalArchiveResultSchema(OwnedObjectSchema):
-    TYPE: str = "core.models.ArchiveResult"
-    id: UUID
-    created_at: datetime | None
-    modified_at: datetime | None
-    status: str
-    retry_at: datetime | None = None
-    plugin: str
-    hook_name: str
-    process_id: UUID | None
-    cmd_version: str | None
-    cmd: list[str] | None
-    pwd: str | None
-    output_str: str
-    output_json: dict[str, Any] | None
-    output_files: dict[str, dict[str, Any]] | None
-    output_size: int
-    output_mimetypes: str
-    start_ts: datetime | None
-    end_ts: datetime | None
-
-    @staticmethod
-    def resolve_output_files(obj):
-        return obj.output_file_map()
-
-    @staticmethod
-    def resolve_output_mimetypes(obj) -> str:
-        mime_sizes: dict[str, int] = defaultdict(int)
-        for metadata in obj.output_file_map().values():
-            if not isinstance(metadata, dict):
-                continue
-            mimetype = str(metadata.get("mimetype") or "").strip()
-            try:
-                size = max(int(metadata.get("size") or 0), 0)
-            except (TypeError, ValueError):
-                size = 0
-            if mimetype and size:
-                mime_sizes[mimetype] += size
-        if mime_sizes:
-            return ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-        return obj.output_mimetypes or ""
-
-
-class ArchiveResultSchema(MinimalArchiveResultSchema):
-    TYPE: str = "core.models.ArchiveResult"
-    snapshot_id: UUID
-    snapshot_timestamp: str
-    snapshot_url: str
-    snapshot_tags: list[str]
-
-    @staticmethod
-    def resolve_snapshot_timestamp(obj):
-        return obj.snapshot.timestamp
-
-    @staticmethod
-    def resolve_snapshot_url(obj):
-        return obj.snapshot.url
-
-    @staticmethod
-    def resolve_snapshot_id(obj):
-        return obj.snapshot_id
-
-    @staticmethod
-    def resolve_snapshot_tags(obj):
-        return sorted(tag.name for tag in obj.snapshot.tags.all())
-
-
-class ArchiveResultFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup(["id__startswith", "snapshot__id__startswith", "snapshot__timestamp__startswith"])] = None
-    search: Annotated[
-        str | None,
-        FilterLookup(
-            [
-                "snapshot__url__icontains",
-                "snapshot__title__icontains",
-                "snapshot__tags__name__icontains",
-                "plugin",
-                "output_str__icontains",
-                "id__startswith",
-                "snapshot__id__startswith",
-                "snapshot__timestamp__startswith",
-            ],
-        ),
-    ] = None
-    snapshot_id: Annotated[str | None, FilterLookup(["snapshot__id__startswith", "snapshot__timestamp__startswith"])] = None
-    snapshot_url: Annotated[str | None, FilterLookup("snapshot__url__icontains")] = None
-    snapshot_tag: Annotated[str | None, FilterLookup("snapshot__tags__name__icontains")] = None
-    status: Annotated[str | None, FilterLookup("status")] = None
-    output_str: Annotated[str | None, FilterLookup("output_str__icontains")] = None
-    plugin: Annotated[str | None, FilterLookup("plugin__icontains")] = None
-    hook_name: Annotated[str | None, FilterLookup("hook_name__icontains")] = None
-    process_id: Annotated[str | None, FilterLookup("process__id__startswith")] = None
-    cmd: Annotated[str | None, FilterLookup("cmd__0__icontains")] = None
-    pwd: Annotated[str | None, FilterLookup("pwd__icontains")] = None
-    cmd_version: Annotated[str | None, FilterLookup("cmd_version")] = None
-    created_at: Annotated[datetime | None, FilterLookup("created_at")] = None
-    created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
-    created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
-
-
-class SnapshotSchema(OwnedObjectSchema):
-    TYPE: str = "core.models.Snapshot"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    status: str
-    retry_at: datetime | None
-    bookmarked_at: datetime
-    downloaded_at: datetime | None
-    url: str
-    tags: list[str]
-    title: str | None
-    timestamp: str
-    archive_path: str
-    archive_size: int
-    output_size: int
-    num_archiveresults: int
-    archiveresults: list[MinimalArchiveResultSchema]
-
-    @staticmethod
-    def resolve_tags(obj):
-        return sorted(tag.name for tag in obj.tags.all())
-
-    @staticmethod
-    def resolve_archive_size(obj):
-        return int(obj.archive_size or 0)
-
-    @staticmethod
-    def resolve_output_size(obj):
-        return SnapshotSchema.resolve_archive_size(obj)
-
-    @staticmethod
-    def resolve_num_archiveresults(obj, context):
-        return obj.archiveresult_set.all().distinct().count()
-
-    @staticmethod
-    def resolve_archiveresults(obj, context):
-        if bool(context["request"].__dict__.get("with_archiveresults", False)):
-            return obj.archiveresult_set.all().distinct()
-        return ArchiveResult.objects.none()
-
-
-class SnapshotUpdateSchema(Schema):
-    action: str | None = None
-    status: str | None = None
-    retry_at: datetime | None = None
-    tags: list[str] | None = None
-
-
-class SnapshotCreateSchema(Schema):
-    url: str
-    crawl_id: str | None = None
-    depth: int = 0
-    title: str | None = None
-    tags: list[str] | None = None
-    status: str | None = None
-
-
-class SnapshotDeleteResponseSchema(Schema):
-    success: bool
-    snapshot_id: str
-    crawl_id: str
-    deleted_count: int
-
-
-class SnapshotFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup(["id__istartswith", "id__iendswith", "timestamp__startswith"])] = None
-    created_by_id: Annotated[str | None, FilterLookup("crawl__created_by_id")] = None
-    created_by_username: Annotated[str | None, FilterLookup("crawl__created_by__username__icontains")] = None
-    created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
-    created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
-    created_at: Annotated[datetime | None, FilterLookup("created_at")] = None
-    modified_at: Annotated[datetime | None, FilterLookup("modified_at")] = None
-    modified_at__gte: Annotated[datetime | None, FilterLookup("modified_at__gte")] = None
-    modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
-    search: str | None = None
-    search_mode: str | None = None
-    status: str | None = None
-    url: Annotated[str | None, FilterLookup("url")] = None
-    tag: Annotated[str | None, FilterLookup("tags__name")] = None
-    title: Annotated[str | None, FilterLookup("title__icontains")] = None
-    timestamp: Annotated[str | None, FilterLookup("timestamp__startswith")] = None
-    bookmarked_at__gte: Annotated[datetime | None, FilterLookup("bookmarked_at__gte")] = None
-    bookmarked_at__lt: Annotated[datetime | None, FilterLookup("bookmarked_at__lt")] = None
-
-    def filter_search(self, value: str | None) -> Q:
-        return Q()
-
-    def filter_search_mode(self, value: str | None) -> Q:
-        return Q()
-
-    def filter_status(self, value: str | None) -> Q:
-        return Q()
-
-
-class TagSchema(OwnedObjectSchema):
-    TYPE: str = "core.models.Tag"
-    id: int
-    modified_at: datetime
-    created_at: datetime
-    name: str
-    num_snapshots: int
-    snapshots: list[SnapshotSchema]
-
-    @staticmethod
-    def resolve_num_snapshots(obj, context):
-        return obj.snapshot_set.all().distinct().count()
-
-    @staticmethod
-    def resolve_snapshots(obj, context):
-        if bool(context["request"].__dict__.get("with_snapshots", False)):
-            return obj.snapshot_set.all().distinct()
-        return Snapshot.objects.none()
-
-
-class TagAutocompleteSchema(Schema):
-    tags: list[dict]
-
-
-class TagCreateSchema(Schema):
-    name: str
-
-
-class TagCreateResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    tag_name: str
-    created: bool
-
-
-class TagSearchSnapshotSchema(Schema):
-    id: str
-    title: str
-    url: str
-    favicon_url: str
-    admin_url: str
-    archive_url: str
-    downloaded_at: str | None = None
-
-
-class TagSearchCardSchema(Schema):
-    id: int
-    name: str
-    slug: str
-    num_snapshots: int
-    filter_url: str
-    edit_url: str
-    export_urls_url: str
-    export_jsonl_url: str
-    rename_url: str
-    delete_url: str
-    snapshots: list[TagSearchSnapshotSchema]
-
-
-class TagSearchResponseSchema(Schema):
-    tags: list[TagSearchCardSchema]
-    sort: str
-    created_by: str
-    year: str
-    has_snapshots: str
-
-
-class TagUpdateSchema(Schema):
-    name: str
-
-
-class TagUpdateResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    tag_name: str
-
-
-class TagDeleteResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    deleted_count: int
-
-
-class TagSnapshotRequestSchema(Schema):
-    snapshot_id: str
-    tag_name: str | None = None
-    tag_id: int | None = None
-
-
-class TagSnapshotResponseSchema(Schema):
-    success: bool
-    tag_id: int
-    tag_name: str
@@ -1,356 +0,0 @@
-from datetime import datetime, time
-from html import unescape
-from typing import Any
-
-from django.contrib.auth import get_user_model
-from django.contrib.auth.models import User
-from django.core.exceptions import ValidationError
-from django.db import IntegrityError
-from django.db.models import Q
-from django.http import HttpRequest, HttpResponse
-from django.utils import timezone
-from django.utils.dateparse import parse_date, parse_datetime
-from django.utils.feedgenerator import Rss201rev2Feed
-from ninja import Query, Router
-from ninja.errors import HttpError
-from ninja.pagination import paginate
-
-from archivebox.api.v1_crawls import get_crawl_by_ref
-from archivebox.core.models import Snapshot
-from archivebox.core.routes_util import build_web_url
-from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
-from archivebox.crawls.locks import crawl_lifecycle_lock
-from archivebox.crawls.models import Crawl
-from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
-from archivebox.search.config import get_search_mode, get_search_mode_backend
-from archivebox.search.query import apply_snapshot_search
-
-from .lookup import _get_snapshot_by_ref
-from .pagination import CustomPagination
-from .schemas import SnapshotCreateSchema, SnapshotDeleteResponseSchema, SnapshotFilterSchema, SnapshotSchema, SnapshotUpdateSchema
-
-router = Router(tags=["Core Models"])
-
-
-def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
-    return [tag.strip() for tag in (tags or []) if tag and tag.strip()]
-
-
-def _parse_rss_before(before: str | None) -> datetime:
-    if not before:
-        return timezone.now()
-
-    value = before.strip()
-    parsed_dt = None
-
-    if len(value) == 8 and value.isdigit():
-        parsed_date = datetime.strptime(value, "%Y%m%d").date()
-    else:
-        parsed_dt = parse_datetime(value)
-        parsed_date = None if parsed_dt else parse_date(value)
-
-    if parsed_dt is None:
-        if parsed_date is None:
-            raise HttpError(400, "before must be an ISO datetime, YYYY-MM-DD, or YYYYMMDD")
-        parsed_dt = datetime.combine(parsed_date, time.max)
-
-    if timezone.is_naive(parsed_dt):
-        parsed_dt = timezone.make_aware(parsed_dt, timezone.get_current_timezone())
-    return parsed_dt
-
-
-def _filter_snapshots_for_rss(
-    *,
-    crawl_id: str = "",
-    created_by: str = "",
-    before: str | None = None,
-    limit: int = 50,
-):
-    limit = max(1, min(int(limit or 50), 500))
-    before_dt = _parse_rss_before(before)
-    queryset = (
-        Snapshot.objects.select_related("crawl__created_by")
-        .prefetch_related("tags")
-        .only(
-            "id",
-            "url",
-            "title",
-            "timestamp",
-            "bookmarked_at",
-            "created_at",
-            "modified_at",
-            "fs_version",
-            "crawl_id",
-            "crawl__id",
-            "crawl__created_by_id",
-            "crawl__created_by__id",
-            "crawl__created_by__username",
-        )
-        .filter(bookmarked_at__lte=before_dt)
-    )
-    crawl_id = crawl_id.strip()
-    if crawl_id:
-        matching_crawl_pks = list(filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).values_list("pk", flat=True)[:100])
-        queryset = queryset.filter(crawl_id__in=matching_crawl_pks)
-
-    created_by = created_by.strip()
-    if created_by:
-        created_by_query = Q(crawl__created_by__username__iexact=created_by)
-        user_model = get_user_model()
-        try:
-            prepared_pk = user_model._meta.pk.get_prep_value(created_by)
-        except (TypeError, ValueError, ValidationError):
-            prepared_pk = None
-        if prepared_pk not in (None, ""):
-            created_by_query |= Q(crawl__created_by_id=prepared_pk)
-        queryset = queryset.filter(created_by_query)
-
-    return queryset.order_by("-bookmarked_at", "-created_at", "-id")[:limit]
-
-
-def _snapshots_rss_response(
-    request: HttpRequest,
-    *,
-    snapshots,
-    title: str = "ArchiveBox Snapshots",
-) -> HttpResponse:
-    web_base_url = build_web_url("/", request=request).rstrip("/")
-    feed_query = request.GET.copy()
-    for sensitive_param in ("api_key", "token", "password"):
-        feed_query.pop(sensitive_param, None)
-    feed_path = request.path
-    feed_url = request.build_absolute_uri(f"{feed_path}?{feed_query.urlencode()}" if feed_query else feed_path)
-
-    feed = Rss201rev2Feed(
-        title=title,
-        link=build_web_url("/public/", request=request),
-        description="Recently added ArchiveBox snapshots.",
-        language="en",
-        feed_url=feed_url,
-    )
-
-    for snapshot in snapshots:
-        archived_url = build_web_url(f"/{snapshot.archive_path_from_db}", request=request)
-        tags = [tag.name for tag in snapshot.tags.all()]
-        crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
-        description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
-        feed.add_item(
-            title=unescape(snapshot.title or snapshot.url),
-            link=archived_url or web_base_url,
-            description=description,
-            unique_id=str(snapshot.id),
-            unique_id_is_permalink=False,
-            pubdate=snapshot.bookmarked_at or snapshot.created_at,
-            updateddate=snapshot.modified_at,
-            author_name=crawl_user.username if crawl_user else None,
-            categories=tags,
-        )
-
-    return HttpResponse(feed.writeString("utf-8"), content_type="application/rss+xml; charset=utf-8")
-
-
-@router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
-@paginate(CustomPagination)
-def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
-    """List all Snapshot entries matching these filters."""
-    setattr(request, "with_archiveresults", with_archiveresults)
-    try:
-        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    queryset = filters.filter(queryset).distinct()
-    query = (filters.search or "").strip()
-    if not query:
-        return queryset
-
-    runtime_config = request.archivebox_config
-    search_mode = get_search_mode(filters.search_mode, config=runtime_config)
-    try:
-        return apply_snapshot_search(
-            queryset,
-            query,
-            search_mode=search_mode,
-            config=runtime_config,
-            include_id_matches=True,
-        )
-    except Exception:
-        if get_search_mode_backend(search_mode, config=runtime_config):
-            return queryset.none()
-        return apply_snapshot_search(queryset, query, search_mode="meta", config=runtime_config, include_id_matches=True)
-
-
-@router.get("/snapshots.rss", url_name="get_snapshots_rss")
-def get_snapshots_rss(
-    request: HttpRequest,
-    crawl_id: str = "",
-    created_by: str = "",
-    limit: int = 50,
-    before: str | None = None,
-):
-    """Return matching snapshots as an RSS feed, newest first."""
-    snapshots = _filter_snapshots_for_rss(
-        crawl_id=crawl_id,
-        created_by=created_by,
-        limit=limit,
-        before=before,
-    )
-    return _snapshots_rss_response(request, snapshots=snapshots)
-
-
-@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
-def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
-    """Get a specific Snapshot by id."""
-    setattr(request, "with_archiveresults", with_archiveresults)
-    return _get_snapshot_by_ref(snapshot_id)
-
-
-@router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
-def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
-    tags = normalize_tag_list(data.tags)
-    try:
-        status = normalize_snapshot_status(data.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    if not data.url.strip():
-        raise HttpError(400, "URL is required")
-    try:
-        validate_url_length(data.url.strip())
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    if data.depth not in (0, 1, 2, 3, 4):
-        raise HttpError(400, "depth must be between 0 and 4")
-
-    if data.crawl_id:
-        crawl = get_crawl_by_ref(data.crawl_id)
-        crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
-        tags = tags or crawl_tags
-    else:
-        crawl = Crawl.objects.create(
-            urls=data.url,
-            max_depth=max(data.depth, 0),
-            tags_str=",".join(tags),
-            status=Crawl.StatusChoices.QUEUED,
-            retry_at=timezone.now(),
-            created_by=request.user if isinstance(request.user, User) else None,
-        )
-
-    # Browser uploads must not wait behind the runner's crawl-wide lifecycle
-    # lock. The unique insert recovery and CAS update below are the request-side
-    # coordination boundary for this idempotent metadata sync.
-    snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-    if snapshot is None:
-        try:
-            snapshot = Snapshot.objects.create(
-                url=data.url,
-                crawl=crawl,
-                depth=data.depth,
-                title=data.title,
-                timestamp=str(timezone.now().timestamp()),
-                status=status or Snapshot.StatusChoices.QUEUED,
-                retry_at=timezone.now(),
-            )
-        except IntegrityError:
-            snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-            if snapshot is None:
-                raise
-
-    for _attempt in range(3):
-        updates: dict[str, Any] = {}
-        if data.title is not None and snapshot.title != data.title:
-            updates["title"] = data.title
-        if status is not None and snapshot.status != status:
-            updates["status"] = status
-        if not updates or snapshot.safe_update(updates, extra_filter={"modified_at": snapshot.modified_at}):
-            break
-    else:
-        raise HttpError(409, "Snapshot changed while metadata was being updated")
-
-    if tags:
-        snapshot.save_tags(
-            tags,
-            created_by=request.user if isinstance(request.user, User) else None,
-        )
-
-    try:
-        snapshot.ensure_crawl_symlink()
-    except Exception:
-        pass
-
-    setattr(request, "with_archiveresults", False)
-    return snapshot
-
-
-@router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
-def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
-    """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    snapshot = _get_snapshot_by_ref(snapshot_id)
-    crawl_id = str(snapshot.crawl_id)
-
-    payload = data.dict(exclude_unset=True)
-    update_fields = ["modified_at"]
-    action = payload.pop("action", None)
-    tags = payload.pop("tags", None)
-
-    if action:
-        if action == "pause":
-            snapshot.pause()
-            setattr(request, "with_archiveresults", False)
-            return snapshot
-        if action in ("resume", "unpause"):
-            snapshot.resume()
-            setattr(request, "with_archiveresults", False)
-            return snapshot
-        if action == "cancel":
-            snapshot.cancel()
-            setattr(request, "with_archiveresults", False)
-            return snapshot
-        raise HttpError(400, f"Invalid action: {action}")
-
-    with crawl_lifecycle_lock(crawl_id):
-        snapshot = _get_snapshot_by_ref(snapshot_id)
-        if "status" in payload:
-            try:
-                snapshot.status = normalize_snapshot_status(payload["status"])
-            except ValueError as err:
-                raise HttpError(400, str(err)) from err
-            if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
-                snapshot.retry_at = None
-            update_fields.append("status")
-
-        if "retry_at" in payload:
-            snapshot.retry_at = payload["retry_at"]
-            update_fields.append("retry_at")
-
-        if tags is not None:
-            snapshot.save_tags(
-                normalize_tag_list(tags),
-                created_by=request.user if isinstance(request.user, User) else None,
-            )
-
-        if payload.get("status") == Snapshot.StatusChoices.SEALED:
-            snapshot.cancel()
-        else:
-            snapshot.save(update_fields=update_fields)
-    setattr(request, "with_archiveresults", False)
-    return snapshot
-
-
-@router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
-def delete_snapshot(request: HttpRequest, snapshot_id: str):
-    snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
-    snapshot_id_str = str(snapshot.id)
-    crawl_id_str = str(snapshot.crawl.pk)
-    snapshot.cancel()
-
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
-        deleted_count, _ = snapshot.delete()
-    return {
-        "success": True,
-        "snapshot_id": snapshot_id_str,
-        "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
-    }
@@ -1,285 +0,0 @@
-from django.core.exceptions import ValidationError
-from django.db.models import Q
-from django.http import HttpRequest, HttpResponse
-from ninja import Router
-from ninja.errors import HttpError
-from ninja.pagination import paginate
-
-from archivebox.api.auth import authenticated_user_from_request
-from archivebox.config.common import get_config
-from archivebox.core.models import Snapshot, Tag
-from archivebox.core.permissions import public_snapshots_queryset
-from archivebox.core.tag_util import (
-    add_snapshot_counts,
-    build_tag_cards,
-    export_tag_snapshots_jsonl,
-    export_tag_urls,
-    get_matching_tags,
-    get_tag_by_ref,
-    normalize_created_by_filter,
-    normalize_created_year_filter,
-    normalize_has_snapshots_filter,
-    normalize_tag_sort,
-)
-
-from .pagination import CustomPagination
-from .schemas import (
-    TagAutocompleteSchema,
-    TagCreateResponseSchema,
-    TagCreateSchema,
-    TagDeleteResponseSchema,
-    TagSchema,
-    TagSearchResponseSchema,
-    TagSnapshotRequestSchema,
-    TagSnapshotResponseSchema,
-    TagUpdateResponseSchema,
-    TagUpdateSchema,
-)
-
-router = Router(tags=["Core Models"])
-
-
-@router.get("/tags", response=list[TagSchema], url_name="get_tags")
-@paginate(CustomPagination)
-def get_tags(request: HttpRequest):
-    setattr(request, "with_snapshots", False)
-    setattr(request, "with_archiveresults", False)
-    return get_matching_tags()
-
-
-@router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
-def get_tag(request: HttpRequest, tag_id: str, with_snapshots: bool = True):
-    setattr(request, "with_snapshots", with_snapshots)
-    setattr(request, "with_archiveresults", False)
-    try:
-        return get_tag_by_ref(tag_id)
-    except (Tag.DoesNotExist, ValidationError):
-        raise HttpError(404, "Tag not found")
-
-
-def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot:
-    snapshot_ref = str(snapshot_ref or "").strip().lower()
-    if not snapshot_ref:
-        raise HttpError(400, "Snapshot id is required")
-
-    snapshot_qs = Snapshot.objects.only("id")
-    is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref)
-    if is_full_uuid:
-        try:
-            return snapshot_qs.get(pk=snapshot_ref.replace("-", ""))
-        except (Snapshot.DoesNotExist, ValueError):
-            pass
-
-    if len(snapshot_ref) >= 14:
-        try:
-            return snapshot_qs.get(timestamp=snapshot_ref)
-        except Snapshot.DoesNotExist:
-            pass
-        except Snapshot.MultipleObjectsReturned:
-            snapshot = snapshot_qs.filter(timestamp=snapshot_ref).first()
-            if snapshot is not None:
-                return snapshot
-
-    try:
-        return snapshot_qs.get(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref))
-    except Snapshot.DoesNotExist:
-        raise HttpError(404, "Snapshot not found") from None
-    except Snapshot.MultipleObjectsReturned:
-        snapshot = snapshot_qs.filter(Q(id__startswith=snapshot_ref) | Q(timestamp__startswith=snapshot_ref)).first()
-        if snapshot is None:
-            raise HttpError(404, "Snapshot not found")
-        return snapshot
-
-
-@router.get("/tags/search/", response=TagSearchResponseSchema, url_name="search_tags")
-def search_tags(
-    request: HttpRequest,
-    q: str = "",
-    sort: str = "created_desc",
-    created_by: str = "",
-    year: str = "",
-    has_snapshots: str = "all",
-):
-    """Return detailed tag cards for admin/live-search UIs."""
-    normalized_sort = normalize_tag_sort(sort)
-    normalized_created_by = normalize_created_by_filter(created_by)
-    normalized_year = normalize_created_year_filter(year)
-    normalized_has_snapshots = normalize_has_snapshots_filter(has_snapshots)
-    return {
-        "tags": build_tag_cards(
-            query=q,
-            request=request,
-            preview_limit=0,
-            sort=normalized_sort,
-            created_by=normalized_created_by,
-            year=normalized_year,
-            has_snapshots=normalized_has_snapshots,
-        ),
-        "sort": normalized_sort,
-        "created_by": normalized_created_by,
-        "year": normalized_year,
-        "has_snapshots": normalized_has_snapshots,
-    }
-
-
-def _public_tag_listing_enabled() -> bool:
-    return get_config().PUBLIC_INDEX
-
-
-def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    if authenticated_user_from_request(request):
-        return True
-
-    return _public_tag_listing_enabled()
-
-
-@router.get("/tags/autocomplete/", response=TagAutocompleteSchema, url_name="tags_autocomplete", auth=None)
-def tags_autocomplete(request: HttpRequest, q: str = ""):
-    """Return tags matching the query for autocomplete."""
-    if not _request_has_tag_autocomplete_access(request):
-        raise HttpError(401, "Authentication required")
-
-    public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
-    queryset = get_matching_tags(q)
-    public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
-    if public_only:
-        queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct()
-    tags = list(queryset[: 50 if not q else 20])
-    add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
-
-    return {
-        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
-    }
-
-
-@router.post("/tags/create/", response=TagCreateResponseSchema, url_name="tags_create")
-def tags_create(request: HttpRequest, data: TagCreateSchema):
-    """Create a new tag or return existing one."""
-    try:
-        tag, created = Tag.get_or_create_by_name(
-            data.name,
-            created_by=request.user if request.user.is_authenticated else None,
-        )
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-        "created": created,
-    }
-
-
-@router.post("/tag/{tag_id}/rename", response=TagUpdateResponseSchema, url_name="rename_tag")
-def rename_tag(request: HttpRequest, tag_id: int, data: TagUpdateSchema):
-    try:
-        tag = get_tag_by_ref(tag_id).rename(data.name)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-    }
-
-
-@router.delete("/tag/{tag_id}", response=TagDeleteResponseSchema, url_name="delete_tag")
-def delete_tag(request: HttpRequest, tag_id: int):
-    try:
-        tag = get_tag_by_ref(tag_id)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-
-    deleted_count, _ = tag.delete()
-    return {
-        "success": True,
-        "tag_id": int(tag_id),
-        "deleted_count": deleted_count,
-    }
-
-
-@router.get("/tag/{tag_id}/urls.txt", url_name="tag_urls_export")
-def tag_urls_export(request: HttpRequest, tag_id: int):
-    try:
-        tag = get_tag_by_ref(tag_id)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-
-    response = HttpResponse(export_tag_urls(tag), content_type="text/plain; charset=utf-8")
-    response["Content-Disposition"] = f'attachment; filename="tag-{tag.slug}-urls.txt"'
-    return response
-
-
-@router.get("/tag/{tag_id}/snapshots.jsonl", url_name="tag_snapshots_export")
-def tag_snapshots_export(request: HttpRequest, tag_id: int):
-    try:
-        tag = get_tag_by_ref(tag_id)
-    except Tag.DoesNotExist as err:
-        raise HttpError(404, "Tag not found") from err
-
-    response = HttpResponse(export_tag_snapshots_jsonl(tag), content_type="application/x-ndjson; charset=utf-8")
-    response["Content-Disposition"] = f'attachment; filename="tag-{tag.slug}-snapshots.jsonl"'
-    return response
-
-
-@router.post("/tags/add-to-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_add_to_snapshot")
-def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
-    """Add a tag to a snapshot. Creates the tag if it doesn't exist."""
-    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
-
-    # Get or create the tag
-    if data.tag_name:
-        try:
-            tag, _ = Tag.get_or_create_by_name(
-                data.tag_name,
-                created_by=request.user if request.user.is_authenticated else None,
-            )
-        except ValueError as err:
-            raise HttpError(400, str(err)) from err
-    elif data.tag_id:
-        try:
-            tag = get_tag_by_ref(data.tag_id)
-        except Tag.DoesNotExist:
-            raise HttpError(404, "Tag not found")
-    else:
-        raise HttpError(400, "Either tag_name or tag_id is required")
-
-    snapshot.add_tag_ids([tag.pk])
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-    }
-
-
-@router.post("/tags/remove-from-snapshot/", response=TagSnapshotResponseSchema, url_name="tags_remove_from_snapshot")
-def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
-    """Remove a tag from a snapshot."""
-    snapshot = _get_snapshot_for_tag_edit(data.snapshot_id)
-
-    # Get the tag
-    if data.tag_id:
-        try:
-            tag = Tag.objects.get(pk=data.tag_id)
-        except Tag.DoesNotExist:
-            raise HttpError(404, "Tag not found")
-    elif data.tag_name:
-        try:
-            tag = Tag.objects.get(name__iexact=data.tag_name.strip())
-        except Tag.DoesNotExist:
-            raise HttpError(404, "Tag not found")
-    else:
-        raise HttpError(400, "Either tag_name or tag_id is required")
-
-    snapshot.remove_tag_ids([tag.pk])
-
-    return {
-        "success": True,
-        "tag_id": tag.pk,
-        "tag_name": tag.name,
-    }
@@ -1,258 +0,0 @@
-__package__ = "archivebox.api"
-
-from pathlib import Path
-from uuid import UUID
-from datetime import datetime
-from django.http import FileResponse, HttpRequest
-from django.shortcuts import redirect
-from django.utils import timezone
-
-from django.contrib.auth.models import User
-
-from archivebox.api.schemas import OwnedObjectSchema
-
-from ninja import Router, Schema
-from ninja.errors import HttpError
-
-from archivebox.core.models import Snapshot
-from archivebox.core.permissions import (
-    PERMISSIONS_PUBLIC,
-    PERMISSIONS_UNLISTED,
-    is_admin_user,
-    normalize_permissions,
-)
-from archivebox.config.common import get_config
-from archivebox.crawls.models import Crawl
-from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url
-
-from .auth import API_AUTH_METHODS, authenticated_user_from_request
-
-router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
-
-
-class CrawlSchema(OwnedObjectSchema):
-    TYPE: str = "crawls.models.Crawl"
-
-    id: UUID
-
-    modified_at: datetime
-    created_at: datetime
-
-    status: str
-    retry_at: datetime | None
-    is_paused: bool
-
-    urls: str
-    max_depth: int
-    tags_str: str
-    config: dict
-
-    @staticmethod
-    def resolve_config(obj):
-        # Redact credential values so REST responses can never leak the raw
-        # token/secret/api-key that the operator stored in Crawl.config.
-        from archivebox.config.common import redact_sensitive_config
-
-        return redact_sensitive_config(obj.config)
-
-    @staticmethod
-    def resolve_snapshots(obj, context):
-        if bool(context["request"].__dict__.get("with_snapshots", False)):
-            return obj.snapshot_set.all().distinct()
-        return Snapshot.objects.none()
-
-
-class CrawlUpdateSchema(Schema):
-    action: str | None = None
-    status: str | None = None
-    retry_at: datetime | None = None
-    tags: list[str] | None = None
-    tags_str: str | None = None
-
-
-class CrawlCreateSchema(Schema):
-    urls: list[str]
-    max_depth: int = 0
-    tags: list[str] | None = None
-    tags_str: str = ""
-    label: str = ""
-    notes: str = ""
-    config: dict = {}
-
-
-class CrawlDeleteResponseSchema(Schema):
-    success: bool
-    crawl_id: str
-    deleted_count: int
-    deleted_snapshots: int
-
-
-def normalize_tag_list(tags: list[str] | None = None, tags_str: str = "") -> list[str]:
-    if tags is not None:
-        return [tag.strip() for tag in tags if tag and tag.strip()]
-    return [tag.strip() for tag in tags_str.split(",") if tag.strip()]
-
-
-@router.get("/crawls", response=list[CrawlSchema], url_name="get_crawls")
-def get_crawls(request: HttpRequest):
-    return Crawl.objects.all().distinct()
-
-
-@router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
-def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
-    try:
-        urls = [validate_url(url) for url in data.urls if url and url.strip()]
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    if not urls:
-        raise HttpError(400, "At least one URL is required")
-    if data.max_depth not in (0, 1, 2, 3, 4):
-        raise HttpError(400, "max_depth must be between 0 and 4")
-
-    tags = normalize_tag_list(data.tags, data.tags_str)
-    config = dict(data.config or {})
-    config.setdefault("PARSER", "url_list")
-    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
-    crawl = Crawl.objects.create(
-        urls="\n".join(urls),
-        max_depth=data.max_depth,
-        tags_str=",".join(tags),
-        label=data.label,
-        notes=data.notes,
-        config=config,
-        status=Crawl.StatusChoices.QUEUED,
-        retry_at=timezone.now(),
-        created_by=request.user if isinstance(request.user, User) else None,
-    )
-    return crawl
-
-
-def get_crawl_by_ref(crawl_id: str):
-    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
-
-
-@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
-def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
-    """Get a specific Crawl by id."""
-    setattr(request, "with_snapshots", with_snapshots)
-    setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = get_crawl_by_ref(crawl_id)
-
-    if crawl and as_rss:
-        query = request.GET.copy()
-        query.pop("as_rss", None)
-        query["crawl_id"] = str(crawl.id)
-        return redirect(f"/api/v1/core/snapshots.rss?{query.urlencode()}")
-
-    return crawl
-
-
-def crawl_file(request: HttpRequest, crawl_id: str, path: str):
-    # Try to resolve the crawl first; if it doesn't exist, return 404.
-    try:
-        crawl = get_crawl_by_ref(crawl_id)
-    except Crawl.DoesNotExist:
-        raise HttpError(404, "Crawl not found")
-
-    user = authenticated_user_from_request(request)
-
-    # Gate access using the same model as SnapshotView/can_view_snapshot:
-    # admins always pass; owners can see their own crawls; otherwise the crawl
-    # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
-    if not is_admin_user(request):
-        permissions = normalize_permissions(crawl.permissions)
-        is_owner = bool(user and crawl.created_by_id == user.id)
-        if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
-            raise HttpError(404, "Crawl not found")
-
-    crawl_root = Path(crawl.output_dir).resolve()
-    file_path = (crawl_root / path).resolve()
-    if not file_path.is_file() or crawl_root not in file_path.parents:
-        raise HttpError(404, "Crawl file not found")
-
-    response = FileResponse(file_path.open("rb"))
-    response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
-    response["Pragma"] = "no-cache"
-    response["Expires"] = "0"
-    response["X-Content-Type-Options"] = "nosniff"
-    return response
-
-
-@router.get("/crawl/{crawl_id}/files/{filename}", auth=None, url_name="crawl_file_root")
-def crawl_file_root(request: HttpRequest, crawl_id: str, filename: str):
-    return crawl_file(request, crawl_id, filename)
-
-
-@router.get("/crawl/{crawl_id}/files/{folder}/{filename}", auth=None, url_name="crawl_file_nested_1")
-def crawl_file_nested_1(request: HttpRequest, crawl_id: str, folder: str, filename: str):
-    return crawl_file(request, crawl_id, f"{folder}/{filename}")
-
-
-@router.get("/crawl/{crawl_id}/files/{folder}/{subfolder}/{filename}", auth=None, url_name="crawl_file_nested_2")
-def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfolder: str, filename: str):
-    return crawl_file(request, crawl_id, f"{folder}/{subfolder}/{filename}")
-
-
-@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
-def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
-    """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = get_crawl_by_ref(crawl_id)
-    payload = data.dict(exclude_unset=True)
-    update_fields = ["modified_at"]
-
-    action = payload.pop("action", None)
-    if action:
-        if action == "pause":
-            crawl.pause()
-            return crawl
-        if action in ("resume", "unpause"):
-            crawl.resume()
-            return crawl
-        if action == "cancel":
-            crawl.cancel()
-            return crawl
-        raise HttpError(400, f"Invalid action: {action}")
-
-    tags = payload.pop("tags", None)
-    tags_str = payload.pop("tags_str", None)
-    if tags is not None or tags_str is not None:
-        crawl.tags_str = ",".join(normalize_tag_list(tags, tags_str or ""))
-        update_fields.append("tags_str")
-
-    if "status" in payload:
-        if payload["status"] not in Crawl.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        if payload["status"] == Crawl.StatusChoices.SEALED:
-            crawl.cancel()
-            return crawl
-        crawl.status = payload["status"]
-        update_fields.append("status")
-
-    if "retry_at" in payload:
-        crawl.retry_at = payload["retry_at"]
-        update_fields.append("retry_at")
-
-    crawl.save(update_fields=update_fields)
-    return crawl
-
-
-@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
-def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = get_crawl_by_ref(crawl_id)
-    crawl_id_str = str(crawl.id)
-    crawl.cancel()
-
-    from archivebox.crawls.locks import crawl_lifecycle_lock
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        crawl = get_crawl_by_ref(crawl_id_str)
-        snapshot_count = crawl.snapshot_set.count()
-        deleted_count, _ = crawl.delete()
-    return {
-        "success": True,
-        "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
-        "deleted_snapshots": snapshot_count,
-    }
@@ -1,161 +0,0 @@
-__package__ = "archivebox.api"
-
-from uuid import UUID
-from typing import Annotated
-from datetime import datetime
-
-from django.http import HttpRequest
-
-from ninja import FilterLookup, FilterSchema, Query, Router, Schema
-from ninja.pagination import paginate
-
-from archivebox.api.v1_core import CustomPagination
-
-
-router = Router(tags=["Machine and Dependencies"])
-
-
-# ============================================================================
-# Machine Schemas
-# ============================================================================
-
-
-class MachineSchema(Schema):
-    """Schema for Machine model."""
-
-    TYPE: str = "machine.Machine"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    guid: str
-    hostname: str
-    hw_in_docker: bool
-    hw_in_vm: bool
-    hw_manufacturer: str
-    hw_product: str
-    hw_uuid: str
-    os_arch: str
-    os_family: str
-    os_platform: str
-    os_release: str
-    os_kernel: str
-    stats: dict
-    num_uses_succeeded: int
-    num_uses_failed: int
-
-
-class MachineFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup("id__startswith")] = None
-    hostname: Annotated[str | None, FilterLookup("hostname__icontains")] = None
-    os_platform: Annotated[str | None, FilterLookup("os_platform__icontains")] = None
-    os_arch: Annotated[str | None, FilterLookup("os_arch")] = None
-    hw_in_docker: Annotated[bool | None, FilterLookup("hw_in_docker")] = None
-    hw_in_vm: Annotated[bool | None, FilterLookup("hw_in_vm")] = None
-    bin_providers: Annotated[str | None, FilterLookup("bin_providers__icontains")] = None
-
-
-# ============================================================================
-# Binary Schemas
-# ============================================================================
-
-
-class BinarySchema(Schema):
-    """Schema for Binary model."""
-
-    TYPE: str = "machine.Binary"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    machine_id: UUID
-    machine_hostname: str
-    name: str
-    binproviders: str
-    binprovider: str
-    abspath: str
-    version: str
-    sha256: str
-    status: str
-    is_valid: bool
-    num_uses_succeeded: int
-    num_uses_failed: int
-
-    @staticmethod
-    def resolve_machine_hostname(obj) -> str:
-        return obj.machine.hostname
-
-    @staticmethod
-    def resolve_is_valid(obj) -> bool:
-        return obj.is_valid
-
-
-class BinaryFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup("id__startswith")] = None
-    name: Annotated[str | None, FilterLookup("name__icontains")] = None
-    binprovider: Annotated[str | None, FilterLookup("binprovider")] = None
-    status: Annotated[str | None, FilterLookup("status")] = None
-    machine_id: Annotated[str | None, FilterLookup("machine_id__startswith")] = None
-    version: Annotated[str | None, FilterLookup("version__icontains")] = None
-
-
-# ============================================================================
-# Machine Endpoints
-# ============================================================================
-
-
-@router.get("/machines", response=list[MachineSchema], url_name="get_machines")
-@paginate(CustomPagination)
-def get_machines(request: HttpRequest, filters: Query[MachineFilterSchema]):
-    """List all machines."""
-    from archivebox.machine.models import Machine
-
-    return filters.filter(Machine.objects.all()).distinct()
-
-
-@router.get("/machine/current", response=MachineSchema, url_name="get_current_machine")
-def get_current_machine(request: HttpRequest):
-    """Get the current machine."""
-    from archivebox.machine.models import Machine
-
-    return Machine.current()
-
-
-@router.get("/machine/{machine_id}", response=MachineSchema, url_name="get_machine")
-def get_machine(request: HttpRequest, machine_id: str):
-    """Get a specific machine by ID."""
-    from archivebox.machine.models import Machine
-    from django.db.models import Q
-
-    return Machine.objects.get(Q(id__startswith=machine_id) | Q(hostname__iexact=machine_id))
-
-
-# ============================================================================
-
-
-# ============================================================================
-# Binary Endpoints
-# ============================================================================
-
-
-@router.get("/binaries", response=list[BinarySchema], url_name="get_binaries")
-@paginate(CustomPagination)
-def get_binaries(request: HttpRequest, filters: Query[BinaryFilterSchema]):
-    """List all binaries."""
-    from archivebox.machine.models import Binary
-
-    return filters.filter(Binary.objects.all().select_related("machine")).distinct()
-
-
-@router.get("/binary/{binary_id}", response=BinarySchema, url_name="get_binary")
-def get_binary(request: HttpRequest, binary_id: str):
-    """Get a specific binary by ID."""
-    from archivebox.machine.models import Binary
-
-    return Binary.objects.select_related("machine").get(id__startswith=binary_id)
-
-
-@router.get("/binary/by-name/{name}", response=list[BinarySchema], url_name="get_binaries_by_name")
-def get_binaries_by_name(request: HttpRequest, name: str):
-    """Get all binaries with the given name."""
-    from archivebox.machine.models import Binary
-
-    return list(Binary.objects.filter(name__iexact=name).select_related("machine"))
@@ -1,160 +0,0 @@
-__package__ = "archivebox.api"
-
-import json
-from datetime import datetime
-from typing import Any
-from uuid import UUID
-
-from django.db.models import Q
-from django.http import HttpRequest
-from archivebox.api.schemas import OwnedObjectSchema
-
-from ninja import Router, Schema
-from ninja.pagination import paginate
-from pydantic import Field
-
-from archivebox.api.v1_core import CustomPagination
-from archivebox.personas.importers import validate_persona_name
-from archivebox.personas.models import Persona
-
-
-router = Router(tags=["Personas"])
-
-
-class PersonaBrowserSettingsSchema(Schema):
-    user_agent: str = ""
-    viewport_size: str = ""
-    viewport_device_scale_factor: float | None = None
-    language: str = ""
-    timezone: str = ""
-    geolocation: dict[str, Any] | None = None
-
-
-class PersonaSyncSchema(Schema):
-    extension_persona_id: str
-    name: str
-    settings: PersonaBrowserSettingsSchema = Field(default_factory=PersonaBrowserSettingsSchema)
-    cookies_txt: str = ""
-    auth_json: dict[str, Any] = Field(default_factory=dict)
-
-
-class PersonaSchema(OwnedObjectSchema):
-    TYPE: str = "personas.models.Persona"
-    id: UUID
-    name: str
-    created_at: datetime
-    config: dict[str, Any] | None
-
-    @staticmethod
-    def resolve_config(obj):
-        # Redact credential values so REST responses don't leak the raw
-        # token/secret/api-key the operator stored in Persona.config.
-        from archivebox.config.common import redact_sensitive_config
-
-        return redact_sensitive_config(obj.config)
-
-
-class PersonaSyncResponseSchema(Schema):
-    success: bool
-    created: bool
-    persona: PersonaSchema
-    cookies_file_written: bool
-    auth_file_written: bool
-
-
-def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrowserSettingsSchema) -> dict[str, Any]:
-    config: dict[str, Any] = {
-        "BROWSER_EXTENSION_PERSONA_ID": extension_persona_id,
-        "BROWSER_EXTENSION_SYNCED_AT": datetime.utcnow().isoformat() + "Z",
-    }
-
-    if settings.user_agent:
-        config.update(
-            {
-                "USER_AGENT": settings.user_agent,
-                "CHROME_USER_AGENT": settings.user_agent,
-                "WGET_USER_AGENT": settings.user_agent,
-                "CURL_USER_AGENT": settings.user_agent,
-            },
-        )
-    if settings.viewport_size:
-        config.update(
-            {
-                "RESOLUTION": settings.viewport_size,
-                "CHROME_RESOLUTION": settings.viewport_size,
-            },
-        )
-    if settings.viewport_device_scale_factor is not None:
-        config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
-    if settings.language:
-        config["BROWSER_LANGUAGE"] = settings.language
-    if settings.timezone:
-        config["BROWSER_TIMEZONE"] = settings.timezone
-    if settings.geolocation:
-        config["BROWSER_GEOLOCATION"] = settings.geolocation
-
-    return config
-
-
-def find_persona(extension_persona_id: str, name: str) -> Persona | None:
-    return (
-        Persona.objects.filter(
-            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
-        )
-        .order_by("created_at")
-        .first()
-    )
-
-
-@router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
-@paginate(CustomPagination)
-def get_personas(request: HttpRequest):
-    """List personas available on this ArchiveBox server."""
-    return Persona.objects.all().order_by("name")
-
-
-@router.post("/sync", response=PersonaSyncResponseSchema, url_name="sync_persona")
-def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
-    """
-    Create or update a Persona from a browser extension profile export.
-
-    The extension sends browser settings plus portable auth artifacts. The server
-    keeps browser override settings in Persona.config and writes cookies.txt /
-    auth.json into the persona directory for extractors to consume.
-    """
-    name = payload.name.strip()
-    is_valid, error_message = validate_persona_name(name)
-    if not is_valid:
-        raise ValueError(error_message)
-
-    persona = find_persona(payload.extension_persona_id, name)
-    created = persona is None
-    if persona is None:
-        persona = Persona(name=name)
-        if request.user.is_authenticated:
-            persona.created_by = request.user
-
-    persona.config = {
-        **(persona.config or {}),
-        **browser_settings_to_config(payload.extension_persona_id, payload.settings),
-    }
-    persona.save()
-    persona.ensure_dirs()
-
-    cookies_written = False
-    if payload.cookies_txt.strip():
-        (persona.path / "cookies.txt").write_text(payload.cookies_txt)
-        cookies_written = True
-
-    auth_written = False
-    if payload.auth_json:
-        (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
-        auth_written = True
-
-    return {
-        "success": True,
-        "created": created,
-        "persona": persona,
-        "cookies_file_written": cookies_written,
-        "auth_file_written": auth_written,
-    }
@@ -1,32 +0,0 @@
-__package__ = "archivebox.api"
-
-from typing import Any
-from collections.abc import Callable
-import logging
-
-from django.db import transaction
-from signal_webhooks.handlers import sync_task_handler
-
-
-logger = logging.getLogger(__name__)
-
-
-def warning_error_handler(hook: Any, error: Exception | None) -> None:
-    if error is not None:
-        logger.warning("Outbound webhook %r failed: %s", hook.name, error)
-        return
-
-    logger.warning("Outbound webhook %r returned a non-success response.", hook.name)
-
-
-def transaction_on_commit_task_handler(hook: Callable[..., None], **kwargs: Any) -> None:
-    def run_webhook() -> None:
-        try:
-            sync_task_handler(hook, **kwargs)
-        except Exception:
-            logger.warning("Outbound webhook failed after transaction commit.", exc_info=True)
-
-    try:
-        transaction.on_commit(run_webhook)
-    except Exception:
-        logger.warning("Could not schedule outbound webhook after transaction commit.", exc_info=True)
@@ -1 +0,0 @@
-__package__ = "archivebox.base_models"
@@ -1,349 +0,0 @@
-"""Base admin classes for models using UUIDv7."""
-
-__package__ = "archivebox.base_models"
-
-import json
-import uuid
-from collections.abc import Mapping
-from typing import ClassVar, NotRequired, TypedDict, cast
-
-from django import forms
-from django.contrib import admin
-from django.db import DatabaseError, models
-from django.forms.renderers import BaseRenderer
-from django.http import HttpRequest, QueryDict
-from django.urls import path, register_converter
-from django.utils.html import format_html
-from django.utils.safestring import SafeString, mark_safe
-from django_object_actions import DjangoObjectActions
-
-
-def card_fieldset(title: str | None, fields: tuple, *, wide: bool = False, **options):
-    """Build a Django fieldset using the shared card layout."""
-    return title, {"fields": fields, "classes": ("card", "wide") if wide else ("card",), **options}
-
-
-class HexUUIDConverter:
-    """URL path converter that canonicalizes UUIDs to their 32-char hex form.
-
-    Accepts both the hyphenated (``aaaaaaaa-bbbb-...``) and bare-hex
-    (``aaaaaaaabbbb...``) UUID strings on the way in (Django's UUIDField
-    parses either), but ``to_url`` always emits the bare-hex form. This is
-    what makes ``reverse("admin:app_model_change", args=[obj.pk])`` produce
-    ``/admin/app/model/06a1a8facb0d.../change/`` instead of the default
-    hyphenated rendering — admin links throughout the app reverse through
-    this converter once ``BaseModelAdmin.get_urls`` swaps in
-    ``<hexuuid:object_id>`` below.
-    """
-
-    regex = r"[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
-
-    def to_python(self, value: str) -> str:
-        # Strip hyphens but stay as a string — Django admin treats the
-        # captured object_id as a string and calls ``model._meta.pk.to_python``
-        # itself, so we don't want to short-circuit that.
-        return value.replace("-", "")
-
-    def to_url(self, value) -> str:
-        if isinstance(value, uuid.UUID):
-            return value.hex
-        return str(value).replace("-", "")
-
-
-register_converter(HexUUIDConverter, "hexuuid")
-
-
-class ConfigOption(TypedDict):
-    plugin: str
-    type: str | list[str]
-    default: object
-    description: str
-    enum: NotRequired[list[object]]
-    pattern: NotRequired[str]
-    minimum: NotRequired[int | float]
-    maximum: NotRequired[int | float]
-
-
-class KeyValueWidget(forms.Widget):
-    """
-    A widget that renders JSON dict as editable key-value input fields
-    with + and - buttons to add/remove rows.
-    Includes autocomplete for available config keys from the plugin system.
-    """
-
-    template_name = ""  # We render manually
-
-    class Media:
-        css: ClassVar[dict[str, list[str]]] = {
-            "all": [],
-        }
-        js: ClassVar[list[str]] = []
-
-    def _get_config_options(self) -> dict[str, ConfigOption]:
-        """Get available config options from plugins."""
-        try:
-            from archivebox.config.common import config_field_metadata
-
-            options: dict[str, ConfigOption] = {}
-            for key, metadata in config_field_metadata().items():
-                option_type = metadata.get("type", "string")
-                # Core fields expose Python type names; plugins use JSON Schema.
-                # The editor should apply the same typed validation to both.
-                type_names = {"bool": "boolean", "int": "integer", "float": "number", "str": "string", "dict": "object", "list": "array"}
-                if isinstance(option_type, str):
-                    option_type = type_names.get(option_type, option_type)
-                option: ConfigOption = {
-                    "plugin": str(metadata.get("plugin", "archivebox")),
-                    "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
-                    "default": metadata.get("default", ""),
-                    "description": str(metadata.get("description", "")),
-                }
-                schema = metadata.get("schema")
-                if isinstance(schema, Mapping):
-                    for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in schema:
-                            option[schema_key] = schema[schema_key]
-                options[key] = option
-            return options
-        except (ImportError, KeyError, TypeError, ValueError):
-            return {}
-
-    def _parse_value(self, value: object) -> dict[str, object]:
-        # Parse JSON value to dict
-        if value is None:
-            return {}
-        if isinstance(value, str):
-            try:
-                parsed = json.loads(value) if value else {}
-            except json.JSONDecodeError:
-                return {}
-            return parsed if isinstance(parsed, dict) else {}
-        if isinstance(value, Mapping):
-            return {str(key): item for key, item in value.items()}
-        return {}
-
-    def render(
-        self,
-        name: str,
-        value: object,
-        attrs: Mapping[str, str] | None = None,
-        renderer: BaseRenderer | None = None,
-    ) -> SafeString:
-        from django.template.loader import render_to_string
-
-        widget_id = attrs.get("id", name) if attrs else name
-        options = self._get_config_options()
-        rows = [
-            mark_safe(self._render_row(widget_id, key, val if isinstance(val, str) else json.dumps(val)))
-            for key, val in self._parse_value(value).items()
-        ]
-        empty_row = mark_safe(self._render_row(widget_id, "", ""))
-        # JSON is embedded in a script, so escape HTML delimiters without changing data.
-        metadata = json.dumps(options).translate({ord("<"): r"\u003C", ord(">"): r"\u003E", ord("&"): r"\u0026"})
-        return mark_safe(
-            render_to_string(
-                "admin/widgets/config.html",
-                {
-                    "name": name,
-                    "widget_id": widget_id,
-                    "datalist_options": mark_safe(
-                        "\n".join(
-                            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
-                            for key, opt in sorted(options.items())
-                        ),
-                    ),
-                    "config_meta_json": metadata,
-                    "rows": [*rows, empty_row],
-                    "empty_row": empty_row,
-                },
-            ),
-        )
-
-    def _render_row(self, widget_id: str, key: str, value: str) -> str:
-        from archivebox.config.common import is_sensitive_config_key
-
-        # Sensitive keys (``*TOKEN*``, ``*SECRET*``, ``*API_KEY*``, ``*APIKEY*``) are
-        # rendered write-only: the input is a password field with a placeholder
-        # showing the value is set, but the raw value is NEVER sent to the browser.
-        # When the user submits the form with the field left blank, the
-        # ``ConfigEditorMixin.save_model`` hook re-merges the previously-saved
-        # value so leaving it untouched is a no-op rather than a destructive clear.
-        is_sensitive = is_sensitive_config_key(key)
-        has_value = bool(value)
-        if is_sensitive:
-            input_type = "password"
-            rendered_value = ""
-            placeholder = (
-                "•••••• (saved — enter new value to replace, clear by deleting row)" if has_value else "value (will be saved write-only)"
-            )
-            extra_attrs = ' autocomplete="off" data-sensitive="1"' + (' data-had-value="1"' if has_value else "")
-        else:
-            input_type = "text"
-            rendered_value = self._escape(value)
-            placeholder = "value"
-            extra_attrs = ""
-        return f'''
-            <div class="key-value-row" style="margin-bottom: 6px;">
-                <div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
-                    <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
-                           style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <input type="{input_type}" class="kv-value" value="{rendered_value}" placeholder="{self._escape(placeholder)}"{extra_attrs}
-                           style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <datalist class="kv-value-options"></datalist>
-                    <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
-                            style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>
-                </div>
-                <div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>
-            </div>
-        '''
-
-    def _escape(self, s: object) -> str:
-        """Escape HTML special chars in attribute values."""
-        if not s:
-            return ""
-        return str(s).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
-
-    def value_from_datadict(
-        self,
-        data: QueryDict | Mapping[str, object],
-        files: object,
-        name: str,
-    ) -> str:
-        value = data.get(name, "{}")
-        return value if isinstance(value, str) else "{}"
-
-
-class ConfigEditorMixin(admin.ModelAdmin):
-    """
-    Mixin for admin classes with a config JSON field.
-
-    Provides a key-value editor widget with autocomplete for available config keys.
-    """
-
-    def formfield_for_dbfield(
-        self,
-        db_field: models.Field,
-        request: HttpRequest,
-        **kwargs: object,
-    ) -> forms.Field | None:
-        """Use KeyValueWidget for the config JSON field."""
-        if db_field.name == "config":
-            kwargs["widget"] = KeyValueWidget()
-        return super().formfield_for_dbfield(db_field, request, **kwargs)
-
-    def save_model(self, request: HttpRequest, obj, form, change):
-        """Preserve write-only redacted credentials on save.
-
-        The KeyValueWidget renders sensitive keys (``*TOKEN*``, ``*SECRET*``,
-        ``*API_KEY*``, ``*APIKEY*``) with an empty value + password input —
-        the real value never leaves the server. On submit, an empty value
-        for a sensitive key that was previously set means "leave untouched",
-        not "clear it." We honor that here by re-merging the stored value
-        before the row is written. Explicitly removing the row in the UI
-        still clears it (the key is gone from the submitted JSON, so there's
-        nothing to merge over).
-        """
-        from archivebox.config.common import is_sensitive_config_key
-
-        if change and obj.pk and obj.config is not None:
-            try:
-                stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
-            except (AttributeError, DatabaseError, TypeError, ValueError):
-                stored = {}
-            if isinstance(stored, dict):
-                new_config = dict(obj.config or {})
-                for key, new_value in list(new_config.items()):
-                    if not is_sensitive_config_key(key):
-                        continue
-                    if new_value not in (None, "") and new_value != "********":
-                        continue
-                    if key in stored:
-                        new_config[key] = stored[key]
-                obj.config = new_config
-        super().save_model(request, obj, form, change)
-
-
-class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
-    list_display = ("id", "created_at", "created_by")
-    readonly_fields = ("id", "created_at", "modified_at")
-    show_search_mode_selector = False
-    change_form_template = "admin/archivebox_change_form.html"
-
-    @admin.display(description="Health", ordering="health")
-    def health_display(self, obj):
-        h = obj.health
-        color = "green" if h >= 80 else "orange" if h >= 50 else "red"
-        return format_html('<span style="color: {};">{}</span>', color, h)
-
-    def get_ordering_fields(self, request):
-        ordering = request.GET.get("o")
-        if not ordering:
-            return set()
-        fields = set()
-        for part in ordering.split("."):
-            if not part:
-                continue
-            try:
-                idx = abs(int(part)) - 1
-            except ValueError:
-                continue
-            if 0 <= idx < len(self.list_display):
-                fields.add(self.list_display[idx])
-        return fields
-
-    def get_admin_toolbar_actions(self, request, obj):
-        """Return extra action button dicts for the shared change-form toolbar.
-
-        See ``templates/admin/includes/archivebox_toolbar.html`` for the
-        accepted keys. Default: no extras (toolbar renders Save/History/Delete
-        plus any ``django-object-actions`` change_actions).
-        """
-        return []
-
-    def render_change_form(self, request, context, add=False, change=False, form_url="", obj=None):
-        context.setdefault("archivebox_admin_actions", self.get_admin_toolbar_actions(request, obj))
-        return super().render_change_form(request, context, add=add, change=change, form_url=form_url, obj=obj)
-
-    def get_default_search_mode(self) -> str:
-        # The shared changelist template always asks every admin for a default
-        # search mode, even when the search-mode toggle is hidden.
-        return "meta"
-
-    def get_form(
-        self,
-        request: HttpRequest,
-        obj: models.Model | None = None,
-        change: bool = False,
-        **kwargs: object,
-    ):
-        form = super().get_form(request, obj, change=change, **kwargs)
-        if "created_by" in form.base_fields:
-            form.base_fields["created_by"].initial = request.user
-        return form
-
-    def get_urls(self):
-        """Swap the per-object admin URLs from ``<path:object_id>`` to
-        ``<hexuuid:object_id>`` so canonical change/delete/history URLs use the
-        32-char hex form. The hyphenated form still resolves because the
-        converter's regex accepts both — Django reverses through ``to_url``
-        which always emits hex, so links in templates / changelists / inline
-        formsets all canonicalize automatically.
-
-        Non-UUID PKs (an ``IntegerField`` PK on some legacy table, for example)
-        won't match the converter's regex and fall back to the default
-        ``<path:object_id>`` patterns we still include after our swap.
-        """
-        info = self.opts.app_label, self.opts.model_name
-        object_routes = [
-            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="{}_{}_history".format(*info)),
-            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="{}_{}_delete".format(*info)),
-            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="{}_{}_change".format(*info)),
-        ]
-        # Append after super().get_urls() so our patterns are the
-        # *last-registered* ones with the canonical admin URL names — Django's
-        # reverse() picks the later registration when names collide, which is
-        # how we make ``reverse("admin:app_model_change", args=[obj.pk])``
-        # emit the hex form. The original ``<path:object_id>`` routes stay in
-        # place as a fallback for non-UUID PKs and for resolving inbound
-        # hyphenated URLs (the ``hexuuid`` regex accepts both forms anyway).
-        return super().get_urls() + object_routes
@@ -1,301 +0,0 @@
-"""Base models using UUIDv7 for all id fields."""
-
-__package__ = "archivebox.base_models"
-
-import json
-import shutil
-from typing import TYPE_CHECKING, Any
-
-from archivebox.uuid_compat import CompactUUIDField, uuid7
-from pathlib import Path
-
-from django.db import models
-from django.db.models import F
-from django.db import transaction
-from django.db.models.signals import pre_delete
-from django.utils import timezone
-from django.contrib.auth import get_user_model
-from django.urls import reverse_lazy
-from django.conf import settings
-
-from django_stubs_ext.db.models import TypedModelMeta
-
-from archivebox.config import CONSTANTS
-
-
-if TYPE_CHECKING:
-    from archivebox.config.common import ArchiveBoxBaseConfig
-
-
-def normalize_config_json_values(config: Any) -> Any:
-    if not isinstance(config, dict):
-        return config
-
-    normalized = dict(config)
-    for key, value in list(normalized.items()):
-        if not isinstance(value, str) or len(value) < 2:
-            continue
-        if value[:1] != '"' or value[-1:] != '"':
-            continue
-        try:
-            decoded = json.loads(value)
-        except ValueError:
-            continue
-        if isinstance(decoded, str):
-            normalized[key] = decoded
-    return normalized
-
-
-def get_or_create_system_user_pk(username="system"):
-    User = get_user_model()
-    # If there's exactly one superuser, use that for all system operations
-    if User.objects.filter(is_superuser=True).count() == 1:
-        return User.objects.filter(is_superuser=True).values_list("pk", flat=True)[0]
-    # Otherwise get or create the system user
-    user, _ = User.objects.get_or_create(
-        username=username,
-        defaults={"is_staff": True, "is_superuser": True, "email": "", "password": "!"},
-    )
-    return user.pk
-
-
-class ModelWithUUID(models.Model):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-    created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL,
-        on_delete=models.CASCADE,
-        default=get_or_create_system_user_pk,
-        null=False,
-        db_index=True,
-    )
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def __str__(self) -> str:
-        return f"[{self.id}] {self.__class__.__name__}"
-
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
-    @property
-    def api_url(self) -> str:
-        return str(reverse_lazy("api-1:get_any", args=[self.id]))
-
-    @property
-    def api_docs_url(self) -> str:
-        return f"/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}"
-
-
-class ModelWithNotes(models.Model):
-    """Mixin for models with a notes field."""
-
-    notes = models.TextField(blank=True, null=False, default="")
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-
-class ModelWithHealthStats(models.Model):
-    """Mixin for models with health tracking fields."""
-
-    num_uses_failed = models.PositiveIntegerField(default=0)
-    num_uses_succeeded = models.PositiveIntegerField(default=0)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
-    @property
-    def health(self) -> int:
-        total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
-        return round((self.num_uses_succeeded / total) * 100)
-
-    def increment_health_stats(self, success: bool):
-        """Atomically increment success or failure counter using F() expression."""
-        field = "num_uses_succeeded" if success else "num_uses_failed"
-        type(self).objects.filter(pk=self.pk).update(
-            **{
-                field: F(field) + 1,
-                "modified_at": timezone.now(),
-            },
-        )
-
-
-class ModelWithConfig(models.Model):
-    """Mixin for models with a JSON config field."""
-
-    _runtime_config: "ArchiveBoxBaseConfig"
-    config = models.JSONField(default=dict, null=True, blank=True, editable=True)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def save(self, *args, **kwargs):
-        normalized_config = normalize_config_json_values(self.config)
-        if normalized_config != self.config:
-            self.config = normalized_config
-            update_fields = kwargs.get("update_fields")
-            if update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
-        super().save(*args, **kwargs)
-
-
-class ModelWithDeleteAfter(models.Model):
-    delete_after_final_statuses: tuple[str, ...] = ()
-    delete_at = models.DateTimeField(default=None, null=True, blank=True, db_index=True)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def save(self, *args, **kwargs):
-        update_fields = kwargs.get("update_fields")
-        if self.delete_at is None:
-            self.set_delete_at_from_config()
-            if self.delete_at is not None and update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
-        super().save(*args, **kwargs)
-
-    def get_delete_after_config_value(self):
-        from archivebox.config.common import get_config
-
-        return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
-
-    def set_delete_at_from_config(self, config_value=None) -> bool:
-        if self.delete_at is not None:
-            return False
-
-        from archivebox.config.common import parse_delete_after
-
-        duration = parse_delete_after(self.get_delete_after_config_value() if config_value is None else config_value)
-        if duration is None:
-            return False
-
-        self.delete_at = (self.created_at or timezone.now()) + duration
-        return True
-
-    @classmethod
-    def missing_delete_at_candidates(cls):
-        return cls.objects.none()
-
-    @classmethod
-    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
-        if backfill_missing:
-            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
-            for obj in missing_delete_at:
-                if obj.set_delete_at_from_config():
-                    cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
-                        delete_at=obj.delete_at,
-                        modified_at=timezone.now(),
-                    )
-
-        # Keep the expiration sweep anchored on delete_at. Some large tables
-        # have millions of final-status rows but almost no retained rows; adding
-        # the status filter before SQLite has narrowed by delete_at can make the
-        # runner scan the hot status index before it claims new work.
-        due_pks = list(
-            cls.objects.filter(delete_at__isnull=False, delete_at__lte=timezone.now())
-            .order_by("delete_at", "pk")
-            .values_list("pk", flat=True)[:batch_size],
-        )
-        if not due_pks:
-            return 0
-
-        queryset = cls.objects.filter(pk__in=due_pks)
-        if cls.delete_after_final_statuses:
-            queryset = queryset.filter(status__in=cls.delete_after_final_statuses)
-
-        count = 0
-        expired = list(queryset.order_by("delete_at", "pk"))
-        for obj in expired:
-            obj.delete()
-            count += 1
-        return count
-
-
-class ModelWithOutputDir(ModelWithUUID):
-    class Meta(ModelWithUUID.Meta):
-        abstract = True
-
-    _delete_signal_registered = False
-
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        output_dir = Path(self.output_dir)
-        # Avoid holding SQLite write transactions open across slow filesystem work.
-        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
-        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
-
-    @property
-    def output_dir_parent(self) -> str:
-        return f"{self._meta.model_name}s"
-
-    @property
-    def output_dir_name(self) -> str:
-        return str(self.id)
-
-    @property
-    def output_dir_str(self) -> str:
-        return f"{self.output_dir_parent}/{self.output_dir_name}"
-
-    @property
-    def output_dir(self) -> Path:
-        raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
-
-    def output_paths_for_delete(self) -> tuple[Path, ...]:
-        return (Path(self.output_dir),)
-
-    @classmethod
-    def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
-        data_dir = CONSTANTS.DATA_DIR.resolve()
-        safe_paths = []
-        for raw_path in paths:
-            path = Path(raw_path)
-            is_safe = False
-            for candidate in (path.absolute(), path.resolve()):
-                try:
-                    candidate.relative_to(data_dir)
-                    is_safe = True
-                    break
-                except ValueError:
-                    continue
-            if not is_safe:
-                raise ValueError(f"Refusing to delete output path outside DATA_DIR: {path}")
-            safe_paths.append(path)
-        return tuple(safe_paths)
-
-    @classmethod
-    def delete_output_paths(cls, paths) -> None:
-        for path in cls.validate_output_paths_for_delete(paths):
-            if path.is_symlink() or path.is_file():
-                path.unlink(missing_ok=True)
-            elif path.is_dir():
-                shutil.rmtree(path, ignore_errors=True)
-
-    def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
-        """Capture output paths before DB deletion and remove them after commit."""
-        paths = self.validate_output_paths_for_delete(self.output_paths_for_delete())
-        transaction.on_commit(lambda: self.delete_output_paths(paths), using=using)
-
-    @classmethod
-    def register_delete_signal(cls) -> None:
-        if cls._delete_signal_registered:
-            return
-
-        def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
-            if not isinstance(instance, ModelWithOutputDir):
-                return
-            instance.schedule_delete_cleanup(using=using)
-
-        pre_delete.connect(
-            schedule_output_dir_cleanup,
-            dispatch_uid="archivebox.output_dir_cleanup_on_delete",
-            weak=False,
-        )
-        cls._delete_signal_registered = True
@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.
@@ -0,0 +1,82 @@
+import sys
+import inspect
+from typing import List, Dict, Any, Optional
+from pathlib import Path
+
+import django
+from django.apps import AppConfig
+from django.core.checks import Tags, Warning, register
+from django.db.backends.sqlite3.base import Database as sqlite3
+
+from pydantic import (
+    Field,
+    SerializeAsAny,
+)
+
+from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
+
+from plugantic.extractors import Extractor, ExtractorName
+from plugantic.plugins import Plugin
+from plugantic.configs import ConfigSet, ConfigSectionName
+from plugantic.replayers import Replayer
+
+
+class PythonBinary(Binary):
+    name: BinName = 'python'
+
+    providers_supported: List[BinProvider] = [EnvProvider()]
+    provider_overrides: Dict[str, Any] = {
+        'env': {
+            'subdeps': \
+                lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
+            'abspath': \
+                lambda: sys.executable,
+            'version': \
+                lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
+        },
+    }
+
+class SqliteBinary(Binary):
+    name: BinName = 'sqlite'
+    providers_supported: List[BinProvider] = [EnvProvider()]
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'env': {
+            'abspath': \
+                lambda: inspect.getfile(sqlite3),
+            'version': \
+                lambda: sqlite3.version,
+        },
+    }
+
+class DjangoBinary(Binary):
+    name: BinName = 'django'
+
+    providers_supported: List[BinProvider] = [EnvProvider()]
+    provider_overrides:  Dict[BinProviderName, ProviderLookupDict] = {
+        'env': {
+            'abspath': \
+                lambda: inspect.getfile(django),
+            'version': \
+                lambda: django.VERSION[:3],
+        },
+    }
+
+
+class BasicReplayer(Replayer):
+    name: str = 'basic'
+
+
+class BasePlugin(Plugin):
+    name: str = 'base'
+    configs: List[SerializeAsAny[ConfigSet]] = []
+    binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
+    extractors: List[SerializeAsAny[Extractor]] = []
+    replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()]
+
+
+PLUGINS = [BasePlugin()]
+
+
+class BaseConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'builtin_plugins.base'
@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.
@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.
@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.
@@ -0,0 +1,94 @@
+from typing import List, Optional
+from pathlib import Path
+
+from django.apps import AppConfig
+from django.core.checks import Tags, Warning, register
+
+from pydantic import (
+    Field,
+    SerializeAsAny,
+)
+
+from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider
+
+from plugantic.extractors import Extractor, ExtractorName
+from plugantic.plugins import Plugin
+from plugantic.configs import ConfigSet, ConfigSectionName
+
+
+###################### Config ##########################
+
+class SinglefileToggleConfig(ConfigSet):
+    section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
+
+    SAVE_SINGLEFILE: bool = True
+
+
+class SinglefileDependencyConfig(ConfigSet):
+    section: ConfigSectionName = 'DEPENDENCY_CONFIG'
+
+    SINGLEFILE_BINARY: str = Field(default='wget')
+    SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
+    SINGLEFILE_EXTRA_ARGS: List[str] = []
+    SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
+
+class SinglefileOptionsConfig(ConfigSet):
+    section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
+
+    # loaded from shared config
+    SINGLEFILE_USER_AGENT: str = Field(default='', alias='USER_AGENT')
+    SINGLEFILE_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
+    SINGLEFILE_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
+    SINGLEFILE_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
+    SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
+
+
+
+DEFAULT_CONFIG = {
+    'CHECK_SSL_VALIDITY': False,
+    'SAVE_SINGLEFILE': True,
+    'TIMEOUT': 120,
+}
+
+PLUGIN_CONFIG = [
+    SinglefileToggleConfig(**DEFAULT_CONFIG),
+    SinglefileDependencyConfig(**DEFAULT_CONFIG),
+    SinglefileOptionsConfig(**DEFAULT_CONFIG),
+]
+
+###################### Binaries ############################
+
+class SinglefileBinary(Binary):
+    name: BinName = 'single-file'
+    providers_supported: List[BinProvider] = [EnvProvider(), NpmProvider()]
+
+
+###################### Extractors ##########################
+
+class SinglefileExtractor(Extractor):
+    name: ExtractorName = 'singlefile'
+    binary: Binary = SinglefileBinary()
+
+    def get_output_path(self, snapshot) -> Path:
+        return Path(snapshot.link_dir) / 'singlefile.html'
+
+
+###################### Plugins #############################
+
+
+class SinglefilePlugin(Plugin):
+    name: str = 'singlefile'
+    configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG]
+    binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()]
+    extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()]
+
+PLUGINS = [SinglefilePlugin()]
+
+###################### Django Apps #########################
+
+class SinglefileConfig(AppConfig):
+    name = 'builtin_plugins.singlefile'
+    verbose_name = 'SingleFile'
+
+    def ready(self):
+        print('Loaded singlefile plugin')
@@ -0,0 +1,66 @@
+name: singlefile
+plugin_version: '0.0.1'
+plugin_spec: '0.0.1'
+
+binaries:
+    singlefile:
+        providers:
+            - env
+            - npm
+
+commands:
+    - singlefile.exec
+    - singlefile.extract
+    - singlefile.should_extract
+    - singlefile.get_output_path
+
+extractors:
+    singlefile:
+        binary: singlefile
+        test: singlefile.should_extract
+        extract: singlefile.extract
+        output_files:
+            - singlefile.html
+
+configs:
+    ARCHIVE_METHOD_TOGGLES:
+        SAVE_SINGLEFILE:
+            type: bool
+            default: true
+
+    DEPENDENCY_CONFIG:
+        SINGLEFILE_BINARY:
+            type: str
+            default: wget
+        SINGLEFILE_ARGS:
+            type: Optional[List[str]]
+            default: null
+        SINGLEFILE_EXTRA_ARGS:
+            type: List[str]
+            default: []
+        SINGLEFILE_DEFAULT_ARGS:
+            type: List[str]
+            default: 
+            - "--timeout={TIMEOUT-10}"
+
+    ARCHIVE_METHOD_OPTIONS:
+        SINGLEFILE_USER_AGENT:
+            type: str
+            default: ""
+            alias: USER_AGENT
+        SINGLEFILE_TIMEOUT:
+            type: int
+            default: 60
+            alias: TIMEOUT
+        SINGLEFILE_CHECK_SSL_VALIDITY:
+            type: bool
+            default: true
+            alias: CHECK_SSL_VALIDITY
+        SINGLEFILE_RESTRICT_FILE_NAMES:
+            type: str
+            default: windows
+            alias: RESTRICT_FILE_NAMES
+        SINGLEFILE_COOKIES_FILE:
+            type: Optional[Path]
+            default: null
+            alias: COOKIES_FILE
Author	SHA1	Message	Date
Nick Sweeting	726bcabd82	fix iterator missing chunk_size	2024-08-26 22:18:10 -07:00
Nick Sweeting	d18418cc22	fix ArchiveResult.output_summary admin formatting	2024-08-26 22:17:57 -07:00