Merge branch 'dev' into plugins-browsertrix

.dockerignore

@@ -1,66 +1,22 @@
 .DS_Store
 ._*
-/.heartbeat.json
-/package.json
-/package-lock.json
-__pycache__
-**/__pycache__
 *.pyc
-*.pyo
-*.py[cod]
-*$py.class
+__pycache__/
 .mypy_cache/
 .pytest_cache/
-.ruff_cache/
-.uv-cache/
-.cache/
-.cache*/
 .github/
 .pdm-build/
-.pdm-build/**
 .pdm-python
 .eggs/
-*.egg-info/
 .git/
-!.git/
-.git/*
-.vscode/
 !.git/HEAD
-!.git/packed-refs
-!.git/refs/
-!.git/refs/heads/
-!.git/refs/heads/**
+!.git/refs/heads/*
 
 venv/
 .venv/
 .venv-old/
-.docker_venv/
 .docker-venv/
 node_modules/
-abx-dl/
-abxpkg/
-abx-plugins/
-abxbus/
-chrome/
-chromeprofile/
-chrome_profile/
-lib/
-out/
-tests/out/
-users/
-archive/
-crawls/
-snapshots/
-logs/
-archivebox-docker-smoke*/
-archivebox-compose-smoke*/
-docker-test/
-docker-test*/
-core
-*.core
-
-pdm.dev.lock
-pdm.lock
 
 docs/
 build/
@@ -68,37 +24,9 @@ dist/
 brew_dist/
 deb_dist/
 pip_dist/
-cache/
-caches/
 assets/
 docker/
-website/
-typings/
 
-# The runtime COPY needs ArchiveBox's package and bin/ entrypoint, not the
-# development UI screenshots or test trees. PNGs are already compressed, so
-# including them creates a large image layer that later cleanup cannot remove.
-publicsite/
-old/
-test/
-tests/
-archivebox/tests/
-
-tmp/
-.tmp/
 data/
 data*/
--
-personas/
-sources/
 output/
-index.sqlite3
-index.sqlite3-wal
-queue.sqlite3
-*.sqlite*
-data.*
-.archivebox_id
-ArchiveBox.conf
-*.stdout
-*.stderr
-*.log

.github/.readthedocs.yaml

@@ -1,26 +0,0 @@
-# Read the Docs config for https://docs.archivebox.io
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
-
-version: 2
-
-submodules:
-  include: all
-  recursive: true
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.12"
-    #nodejs: "20"     # not needed unless we need the full archivebox to run while building docs for some reason
-
-sphinx:
-  configuration: docs/conf.py
-
-formats:
-  - pdf
-  - epub
-
-# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-python:
-  install:
-    - requirements: docs/requirements.txt

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@
 - https://github.com/ArchiveBox/ArchiveBox/issues
 - https://github.com/ArchiveBox/ArchiveBox/pulls
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
 
 ### Development Setup
 

.github/FUNDING.yml

@@ -1,2 +1,3 @@
-github: ["ArchiveBox", "pirate"]
-custom: ["https://donate.archivebox.io", "https://swag.archivebox.io"]
+github: pirate
+patreon: theSquashSH
+custom: ["https://hcb.hackclub.com/donations/start/archivebox", "https://paypal.me/NicholasSweeting"]

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -1,197 +0,0 @@
-name: 🐞 Bug report
-description: Report a bug or error you encountered in ArchiveBox
-title: "Bug: ..."
-assignees:
-  - pirate
-type: 'Bug'
-body:
-  - type: markdown
-    attributes:
-      value: |
-        *Please note:* it is normal to see errors occasionally for some extractors on some URLs (not every extractor will work on every type of page).  
-        Please report archiving errors if you are seeing them *consistently across many URLs* or if they are *preventing you from using ArchiveBox*.
-
-  - type: textarea
-    id: description
-    attributes:
-      label: Provide a screenshot and describe the bug
-      description: |
-        Attach a screenshot and describe what the issue is, what you expected to happen, and if relevant, the *URLs you were trying to archive*.  
-      placeholder: |
-        Got a bunch of 'singlefile was unable to archive this page' errors when trying to archive URLs from this site: https://example.com/xyz ...
-        I also tried to archive the same URLs using `singlefile` directly and some of them worked but not all of them. etc. ...
-    validations:
-      required: true
-
-  - type: textarea
-    id: steps_to_reproduce
-    attributes:
-      label: Steps to reproduce
-      description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
-      render: markdown
-      placeholder: |
-        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 5797:5797 archivebox/archivebox` in iTerm2
-        2. Went to the https://127.0.0.1:5797/add/ page in Google Chrome
-        3. Typed 'https://example.com/xyz' into the 'Add URL' input field
-        4. Clicked the 'Add+' button
-        5. Got a 500 error and saw the errors below in terminal
-    validations:
-      required: true
-
-  - type: textarea
-    id: logs
-    attributes:
-      label: Logs or errors
-      description: "Paste any terminal output, logs, or errors (check `data/logs/errors.log` as well)."
-      placeholder: |
-        ╭─────────────────────────────────────────────────────────────────────────────────────────────────────────╮
-        │ [2024-11-02 19:54:28] ArchiveBox v0.8.6rc0: archivebox add https://example.com#1234567                  │
-        ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯
-        
-        [+] [2024-11-02 19:54:29] Adding 1 links to index (crawl depth=0)...
-            > Saved verbatim input to sources/1730577269-import.txt
-            > Parsed 1 URLs from input (Generic TXT)
-            ...
-      render: shell
-    validations:
-      required: false
-
-  - type: textarea
-    id: version
-    attributes:
-      label: ArchiveBox Version
-      description: |
-        **REQUIRED:** Run the `archivebox version` command inside your collection dir and paste the *full output* here (*not just the version number*).  
-        For Docker Compose run: `docker compose run archivebox version`  
-        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
-      render: shell
-      placeholder: |
-        0.8.6
-        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
-        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
-        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
-        
-        Binary Dependencies:
-        √  python     3.14.0     venv_pip   ~/.venv/bin/python
-        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
-        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
-        ...
-    validations:
-      required: true
-
-  - type: dropdown
-    id: install_method
-    validations:
-      required: true
-    attributes:
-      label: How did you install the version of ArchiveBox you are using?
-      multiple: false
-      options:
-        - pip
-        - apt
-        - brew
-        - nix
-        - Docker (or Podman/LXC/K8s/TrueNAS/Proxmox/etc)
-        - Other
-
-  - type: dropdown
-    id: operating_system
-    validations:
-      required: true
-    attributes:
-      label: What operating system are you running on?
-      description: |
-        Please note we are *unable to provide support for Windows users* unless you are using [Docker on Windows](https://github.com/ArchiveBox/archivebox#:~:text=windows%20without%20docker).
-      multiple: false
-      options:
-        - Linux (Ubuntu/Debian/Arch/Alpine/etc.)
-        - macOS (including Docker on macOS)
-        - BSD (FreeBSD/OpenBSD/NetBSD/etc.)
-        - Windows (including WSL, WSL2, Docker Desktop on Windows)
-        - Other
-
-  - type: checkboxes
-    id: filesystem
-    attributes:
-      label: What type of drive are you using to store your ArchiveBox data?
-      description: Are you using a [remote filesystem](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#supported-remote-filesystems) or FUSE mount for `data/` OR `data/archive`?
-      options:
-        - label: "some of `data/` is on a local SSD or NVMe drive"
-          required: false
-        - label: "some of `data/` is on a spinning hard drive or external USB drive"
-          required: false
-        - label: "some of `data/` is on a network mount (e.g. NFS/SMB/Ceph/GlusterFS/etc.)"
-          required: false
-        - label: "some of `data/` is on a FUSE mount (e.g. SSHFS/RClone/S3/B2/Google Drive/Dropbox/etc.)"
-          required: false
-  
-
-  - type: textarea
-    id: docker_compose_yml
-    attributes:
-      label: Docker Compose Configuration
-      description: "If using Docker Compose, please share your full `docker-compose.yml` file. If using plain Docker, paste the `docker run ...` command you use."
-      placeholder: |
-        services:
-            archivebox:
-                image: archivebox/archivebox:latest
-                ports:
-                    - 5797:5797
-                volumes:
-                    - ./data:/data
-                environment:
-                    - ADMIN_USERNAME=admin
-                    - ADMIN_PASSWORD=****<redact any passwords>****
-                    - ALLOWED_HOSTS=*
-                    - CSRF_TRUSTED_ORIGINS=https://archivebox.example.com
-                    - PUBLIC_INDEX=True
-                    - PUBLIC_SNAPSHOTS=True
-                    - PUBLIC_ADD_VIEW=False
-                    ...
-
-            archivebox_scheduler:
-                image: archivebox/archivebox:latest
-                command: schedule --foreground --update --every=day
-                environment:
-                    ...
-      
-          ...
-      render: shell
-    validations:
-      required: false
-
-  - type: textarea
-    id: configuration
-    attributes:
-      label: ArchiveBox Configuration
-      description: "Please share your full `data/ArchiveBox.conf` file here."
-      render: shell
-      placeholder: |
-        [SERVER_CONFIG]
-        SECRET_KEY = "*********<redact any secrets/passwords>************"
-
-        WGET_RESTRICT_FILE_NAMES=windows
-        USE_SYSTEM_WGET=true
-        CHECK_SSL_VALIDITY=false
-        ...
-    validations:
-      required: false
-
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying anything suggested here, **we are not responsible for data loss**.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
-          - 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
-          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
-        

.github/ISSUE_TEMPLATE/2-feature_request.yml

@@ -1,124 +0,0 @@
-name: 💡 Feature or enhancement request
-description: Suggest an idea or improvement for this project
-title: "Feature Request: ..."
-assignees:
-  - pirate
-type: 'Enhancement'
-labels: 'status: idea phase'
-body:
-  - type: dropdown
-    id: suggestion_type
-    validations:
-      required: true
-    attributes:
-      label: "What type of suggestion are you making?"
-      multiple: false
-      options:
-        - New extractor / type of content to save
-        - Proposing a new feature
-        - Modification of existing behavior
-        - Web UI or UX design improvement
-
-  - type: textarea
-    id: current_problem
-    attributes:
-      label: "What is the problem that your feature request solves?"
-      description: |
-        Describe the problem or need that your feature request solves, feel free to include any screenshots or examples.
-      placeholder: |
-        e.g. I need to be able to archive spanish and french subtitle files from a particular movie site https://example.com/somevideos that's going down soon.
-    validations:
-      required: true
-
-  - type: textarea
-    id: proposed_solution
-    attributes:
-      label: "What is your proposed solution?"
-      description: |
-        Describe the ideal specific solution you'd want, *and whether it fits into any broader scope of changes*.
-      placeholder: |
-        e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
-        The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
-    validations:
-      required: true
-
-  - type: textarea
-    id: workarounds_tried
-    attributes:
-      label: "What hacks or alternative solutions have you tried to solve the problem?"
-      description: |
-        A description of any alternative approaches, workarounds, or other solutions you've considered to fix the problem.
-      placeholder: |
-        e.g. I wait for archivebox to finish archiving the page, then I manually run `yt-dlp --subs <URL>` inside 
-        the `data/archive/<timestamp>/` directory to download the subtitle files and add them to the snapshot folder.
-    validations:
-      required: true
-
-  - type: textarea
-    id: version
-    attributes:
-      label: Share the entire output of the `archivebox version` command for the current version you are using.
-      description: |
-        DO NOT JUST ENTER "the latest version" OR YOUR ISSUE WILL BE CLOSED.  
-        We need to know what version of ArchiveBox and what feature flags you're currently running with in order to contextualize your feature request.  
-        Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in your specific environment.
-          
-        Run the `archivebox version` command inside your current collection dir and paste the *full output* here (*not just the version number*).  
-        For Docker Compose run: `docker compose run archivebox version`  
-        For plain Docker run:   `docker run -v $PWD:/data archivebox/archivebox version`  
-      render: shell
-      placeholder: |
-        0.8.6
-        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
-        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
-        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
-        
-        Binary Dependencies:
-        √  python     3.14.0     venv_pip   ~/.venv/bin/python
-        √  django     6.0        venv_pip   ~/.venv/lib/python3.14/site-packages/django/__init__.py
-        √  sqlite     2.6.0      venv_pip   ~/.venv/lib/python3.14/site-packages/django/db/backends/sqlite3/base.py
-        √  pip        24.3.1     venv_pip   ~/.venv/bin/pip
-        ...
-    validations:
-      required: true
-
-  - type: checkboxes
-    id: priority
-    attributes:
-      label: "How badly do you want this new feature?"
-      options:
-        - label: "It's an urgent deal-breaker, I can't live without it"
-          required: false
-        - label: "It's important to add it in the near-mid term future"
-          required: false
-        - label: "It would be nice to have eventually"
-          required: false
-        - label: "I'm willing to [start a PR](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) to develop this myself"
-          required: false
-        - label: "I have [donated money](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations) to go towards fixing this issue"
-          required: false
-  
-  - type: checkboxes
-    id: satisfaction_survey
-    attributes:
-      label: Mini Survey
-      description: How do you like ArchiveBox so far?
-      options:
-        - label: "I like ArchiveBox so far / would recommend it to a friend"
-          required: false
-        - label: "I've had a lot of difficulty getting ArchiveBox set up"
-          required: false
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to answer issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-        Make sure your `data/` is [**fully backed up**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout) before trying any workarounds or BETAs suggested here, **we are not responsible for data loss**.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
-          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)

.github/ISSUE_TEMPLATE/3-documentation_change.yml

@@ -1,51 +0,0 @@
-name: 📑 Documentation improvement
-description: Submit an idea or correction for the Wiki documentation
-title: "Documentation: ..."
-labels: 'touches: docs'
-type: 'Enhancement'
-assignees:
-  - pirate
-body:
-  - type: markdown
-    attributes:
-      value: |
-        If you prefer, you can submit a [Pull Request](https://github.com/ArchiveBox/docs) on https://github.com/ArchiveBox/docs to edit the docs directly instead.
-
-  - type: input
-    id: page_url
-    validations:
-      required: true
-    attributes:
-      label: "What is the URL of the page you'd like to see improved?"
-      placeholder: e.g. https://github.com/ArchiveBox/docs/wiki/Install
-
-  - type: input
-    id: section_title
-    validations:
-      required: true
-    attributes:
-      label: "What is the title of the relevant section?"
-      placeholder: e.g. Option B. Automatic Setup Script
-
-  - type: textarea
-    id: suggested_edit
-    attributes:
-      label: "What is the suggested edit?"
-      placeholder: |
-        e.g. Please document how to run the automatic setup script for ArchiveBox on TempleOS.  
-        Attach images, screenshots, code snippets, etc. anything you think would help.
-    validations:
-      required: true
-
-  - type: markdown
-    attributes:
-      value: |
-        ---
-          
-        We strive to address issues as quickly as possible, it usually takes us *about a ~week* to respond.  
-          
-        In the meantime please consider:  
-          
-          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
-          - 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
-          - ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,46 @@
+---
+name: 🐞 Bug report
+about: Create a report to help us improve
+title: 'Bug: ...'
+labels: 'bug'
+assignees: ''
+
+---
+
+<!--
+Please fill out the following information, 
+feel free to delete sections if they're not applicable 
+or if long issue templates annoy you.
+(the only required section is the version information)
+-->
+
+#### Describe the bug
+<!--
+A description of what the bug is, 
+what you expected to happen, 
+and any relevant context about issue.
+-->
+
+#### Steps to reproduce
+<!--
+For example:
+1. Ran ArchiveBox with the following config '...'
+2. Saw this output during archiving '....'
+3. UI didn't show the thing I was expecting '....'
+-->
+
+#### Screenshots or log output
+
+<!--
+If applicable, post any relevant screenshots or copy/pasted terminal output from ArchiveBox.
+If you're reporting a parsing / importing error, **you must paste a copy of your redacted import file here**.
+-->
+
+#### ArchiveBox version
+
+<!-- Run the `archivebox version` command locally then copy paste the result here: -->
+```logs
+replace this line with the *full*, unshortened output of running `archivebox version`
+```
+<!-- Tickets without full version info will closed until it is provided,
+we need the full output here to help you solve your issue -->

.github/ISSUE_TEMPLATE/config.yml

@@ -1,8 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: ❓ Ask a question or start a discussion
-    url: https://github.com/ArchiveBox/ArchiveBox/discussions
-    about: "Ask a question, get support, or start a design discussion (to report a problem please use '🐞 Bug report' instead)"
-  - name: 💬 Chat with the dev team & community on Zulip
-    url: https://zulip.archivebox.io
-    about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."

.github/ISSUE_TEMPLATE/documentation_change.md

@@ -0,0 +1,17 @@
+---
+name: 📑 Documentation change
+about: Submit a suggestion for the Wiki documentation
+title: 'Documentation: Improvement request ...'
+labels: ''
+assignees: ''
+
+---
+<!-- If you perfer, you can make a PR to https://github.com/ArchiveBox/docs instead of opening an issue -->
+
+## Wiki Page URL
+<!-- e.g. https://github.com/pirate/ArchiveBox/wiki/Configuration#use_color -->
+
+
+## Suggested Edit
+<!-- e.g. Please add more example usages, or please fix `xyz` typo to be `abc`. -->
+

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,50 @@
+---
+name: 💡 Feature request
+about: Suggest an idea for this project
+title: 'Feature Request: ...'
+labels: 'changes: behavior,status: idea phase'
+assignees: ''
+
+---
+
+<!--
+Please fill out the following information, 
+feel free to delete sections if they're not applicable 
+or if long issue templates annoy you :)
+-->
+
+## Type 
+
+ - [ ] General question or discussion
+ - [ ] Propose a brand new feature
+ - [ ] Request modification of existing behavior or design
+
+## What is the problem that your feature request solves
+<!--
+e.g. I need to be able to archive spanish and french subtitle files  
+from a particular <example.com> movie site that's going down soon.
+-->
+
+## Describe the ideal specific solution you'd want, and whether it fits into any broader scope of changes
+<!--
+e.g. I specifically need a new archive method to look for multilingual subtitle files related to pages.  
+The bigger picture solution is the ability for custom user scripts to be run in a puppeteer context during archiving.
+-->
+
+## What hacks or alternative solutions have you tried to solve the problem?
+<!--
+A clear and concise description of any alternative solutions, 
+workarounds, or other software you've considered using to fix the problem.
+-->
+
+## How badly do you want this new feature?
+
+ - [ ] It's an urgent deal-breaker, I can't live without it
+ - [ ] It's important to add it in the near-mid term future
+ - [ ] It would be nice to have eventually
+
+---
+
+ - [ ] I'm willing to contribute [dev time](https://github.com/ArchiveBox/ArchiveBox#archivebox-development) / [money](https://github.com/sponsors/pirate) to fix this issue
+ - [ ] I like ArchiveBox so far / would recommend it to a friend
+ - [ ] I've had a lot of difficulty getting ArchiveBox set up

.github/ISSUE_TEMPLATE/question_or_discussion.md

@@ -0,0 +1,9 @@
+---
+name: 💬 Question, discussion, or support request
+about: Start a discussion or ask a question about ArchiveBox
+title: 'Question: ...'
+labels: ''
+assignees: ''
+
+---
+

.github/SECURITY.md

@@ -1,68 +0,0 @@
-# Security Policy
-
----
-
-## Security Documentation
-
-Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
-
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
-- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
-- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
-- https://github.com/ArchiveBox/ArchiveBox#archive-layout
-- https://github.com/ArchiveBox/ArchiveBox#archivebox-development
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
-
----
-
-## Reporting a Vulnerability
-
-~We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.~ We no longer offer CVEs or accept private reporting, 95% of the reports we received were low quality AI slop, so now we just ask for normal Github Issues or PRs.
-
-You can contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp) if you have a truly urgent security report, but we do not issue public CVEs.
-
----
-
-## CVEs and Report Credits
-
-> [!IMPORTANT]
-> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**  
-> Reporters **will still receive credit in release notes** however.
-
-> [!TIP]
-> ~Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.~
-
-Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
-
----
-
-## Threat Model
-
-### All admin users are assumed to be trusted
-
-ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
-
-- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
-- non-logged in anonymous guest users
-
-Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
-
-### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
-
-Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
-
-Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
-
-### XSS, CSRF, CORS, CSP
-
-`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
-
-[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
-
-### SSRF / URL Validation
-
-Note archivebox is **designed to be able to archive any URL the server can reach**, including internal intranet URLs, private IPs, and localhost. Users are expected to configure `URL_ALLOWLIST`/`URL_DENYLIST` or proper firewall/ACL-level blocking at the network level if they need to limit what the server should see.
-It does a basic check to try and avoid recursively archiving snapshots hosted by the archivebox server itself, but it's not a strict security check, it's just to prevent wasted CPU/disk from recursively archiving snapshots in a browser that has the ArchiveBox Browser Extension turned on.

.github/STAGING.md

@@ -1,42 +0,0 @@
-# Stable releases require live dev acceptance
-
-The order is **dev CI → dev image publication → automatic Cabbage deployment →
-real capture and browser acceptance → stable publication**. Never
-substitute a published PyPI wheel, healthy container, or local build for this
-acceptance. The release workflow blocks *before* a stable upload unless Cabbage
-has successful evidence from `staging-acceptance.yml` matching all tracked
-source and dependency pins. Only ArchiveBox's own version fields may change
-between dev and stable.
-
-The staging workflow observes the existing image watcher; it cannot deploy or
-restart the application. On Cabbage, install `bin/staging-acceptance-host.sh`
-as root-owned mode 0755 `/usr/local/sbin/archivebox-staging-acceptance`. Its hash
-is checked against the candidate. A dedicated SSH key must use
-`restrict,command="/usr/local/sbin/archivebox-staging-acceptance"` in authorized
-keys. It accepts only `inspect <source-sha>` and `capture <source-sha>`; it cannot
-run caller-provided commands or archive arbitrary URLs.
-
-The `staging` GitHub environment is restricted to dev/main branches. It contains
-`STAGING_SSH_KEY` and `STAGING_KNOWN_HOSTS`, with independently verified host keys.
-Only trusted release events use the private ugNAS runner route. Never expose
-these credentials to PR jobs or copy an operator's existing root key.
-
-Acceptance creates one tagged background crawl per source through the normal
-add/runner path, enabling every snapshot extractor, including TLSNotary. It
-checks installed pins, completed outputs and their files, unexpected/missing
-plugin results, host OOM kills, and real rendered snapshot pages. For this
-release, the explicitly accepted TLSNotary failures and exact missing-Claude-auth
-diagnostics are retained in the report's `limitations` list; they do not disable
-those plugins or excuse missing/truncated successful outputs. Remove the
-TLSNotary exception when its follow-up fix ships. Logs, result inventories and
-screenshots are retained as Actions artifacts. Other failures remain
-failures; do not publish a success deployment record to bypass them. No code
-mounts are allowed. The Cabbage job must pass before its record authorizes a
-stable upload. Queued capture progress is polled; failed tests are not retried.
-
-Browser acceptance follows the default WACZ viewer into the captured document,
-matching its replay base and original URL and checking that its body is visible
-and contains substantial text. An attached viewer iframe can still be
-`about:blank`, so its presence alone cannot prove replay works. Extracted display
-titles and the captured document's `<title>` may legitimately differ; neither
-title equality nor output thumbnails substitute for checking the actual page.

.github/actionlint.yaml

@@ -1,3 +0,0 @@
-self-hosted-runner:
-  labels:
-    - ugnas-linux

.github/configs/ci-linux-build.json

@@ -1,81 +0,0 @@
-{
-  "properties": {
-    "CC_BINARY": {
-      "default": "cc"
-    },
-    "LDAPSEARCH_BINARY": {
-      "default": "ldapsearch"
-    },
-    "INITDB_BINARY": {
-      "default": "initdb"
-    },
-    "PG_CTL_BINARY": {
-      "default": "pg_ctl"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{CC_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "build-essential"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{LDAPSEARCH_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapsearch",
-            "-VV"
-          ]
-        },
-        "apt": {
-          "install_args": [
-            "ldap-utils",
-            "python3-dev",
-            "python3-setuptools",
-            "libssl-dev",
-            "libldap2-dev",
-            "libsasl2-dev",
-            "zlib1g-dev",
-            "libatomic1"
-          ],
-          "version": [
-            "ldapsearch",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ],
-  "postgres_binaries": [
-    {
-      "name": "{INITDB_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "postgresql"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{PG_CTL_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "postgresql"
-          ]
-        }
-      }
-    }
-  ]
-}

.github/configs/ci-macos-brew.json

@@ -1,13 +0,0 @@
-{
-  "properties": {
-    "BREW_BINARY": {
-      "default": "brew"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{BREW_BINARY}",
-      "binproviders": "env"
-    }
-  ]
-}

.github/configs/ci-macos-build.json

@@ -1,44 +0,0 @@
-{
-  "properties": {
-    "CLANG_BINARY": {
-      "default": "clang"
-    },
-    "LDAPVC_BINARY": {
-      "default": "ldapvc"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{CLANG_BINARY}",
-      "binproviders": "env,brew",
-      "overrides": {
-        "brew": {
-          "install_args": [
-            "llvm"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{LDAPVC_BINARY}",
-      "binproviders": "env,brew",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        },
-        "brew": {
-          "install_args": [
-            "openldap"
-          ],
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ]
-}

.github/configs/ci-macos-openldap-env.json

@@ -1,21 +0,0 @@
-{
-  "properties": {
-    "LDAPVC_BINARY": {
-      "default": "ldapvc"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{LDAPVC_BINARY}",
-      "binproviders": "env",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ]
-}

.github/configs/ci-tooling.json

@@ -1,349 +0,0 @@
-{
-  "properties": {
-    "CURL_BINARY": {
-      "default": "curl"
-    },
-    "BASH_BINARY": {
-      "default": "bash"
-    },
-    "CHMOD_BINARY": {
-      "default": "chmod"
-    },
-    "DOCKER_BINARY": {
-      "default": "docker"
-    },
-    "GIT_BINARY": {
-      "default": "git"
-    },
-    "GREP_BINARY": {
-      "default": "grep"
-    },
-    "GH_BINARY": {
-      "default": "gh"
-    },
-    "EXPORTFS_BINARY": {
-      "default": "exportfs"
-    },
-    "JQ_BINARY": {
-      "default": "jq"
-    },
-    "MKDIR_BINARY": {
-      "default": "mkdir"
-    },
-    "MOUNT_CIFS_BINARY": {
-      "default": "mount.cifs"
-    },
-    "MOUNT_NFS_BINARY": {
-      "default": "mount.nfs"
-    },
-    "NODE_BINARY": {
-      "default": "node"
-    },
-    "PS_BINARY": {
-      "default": "ps"
-    },
-    "PYTHON_BINARY": {
-      "default": "python"
-    },
-    "PV_BINARY": {
-      "default": "pv"
-    },
-    "SSH_BINARY": {
-      "default": "ssh"
-    },
-    "SED_BINARY": {
-      "default": "sed"
-    },
-    "SMBD_BINARY": {
-      "default": "smbd"
-    },
-    "SUDO_BINARY": {
-      "default": "sudo"
-    },
-    "SYSTEMCTL_BINARY": {
-      "default": "systemctl"
-    },
-    "TAR_BINARY": {
-      "default": "tar"
-    },
-    "TREE_BINARY": {
-      "default": "tree"
-    },
-    "TR_BINARY": {
-      "default": "tr"
-    },
-    "RM_BINARY": {
-      "default": "rm"
-    },
-    "UMOUNT_BINARY": {
-      "default": "umount"
-    },
-    "UV_BINARY": {
-      "default": "uv"
-    },
-    "UNAME_BINARY": {
-      "default": "uname"
-    }
-  },
-  "ci_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{JQ_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{UV_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "bsd_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,pyinfra"
-    }
-  ],
-  "freebsd_binaries": [
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env,pyinfra"
-    }
-  ],
-  "docker_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UNAME_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "deploy_binaries": [
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SSH_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "docker_debug_binaries": [
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{PV_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{TAR_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{TREE_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "docker_validation_binaries": [
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GREP_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{MKDIR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{RM_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SED_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SSH_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{TR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UNAME_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "mount_validation_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{CHMOD_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{EXPORTFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "nfs-kernel-server"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{MKDIR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{MOUNT_CIFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "cifs-utils"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{MOUNT_NFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "nfs-common"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{SMBD_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "samba"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SYSTEMCTL_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UMOUNT_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "process_binaries": [
-    {
-      "name": "{PS_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "docs_binaries": [
-    {
-      "name": "{CURL_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "git_binaries": [
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "github_binaries": [
-    {
-      "name": "{GH_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "node_binaries": [
-    {
-      "name": "{NODE_BINARY}",
-      "binproviders": "env,npm",
-      "min_version": "22.23.1",
-      "overrides": {
-        "npm": {
-          "install_args": [
-            "node@22.23.1"
-          ],
-          "postinstall_scripts": true
-        }
-      }
-    }
-  ],
-  "release_binaries": [
-    {
-      "name": "{UV_BINARY}",
-      "binproviders": "env,pip,brew"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{JQ_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{CURL_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ]
-}

.github/dependabot.yml

@@ -5,21 +5,8 @@
 
 version: 2
 updates:
-  - package-ecosystem: "pip"
+  - package-ecosystem: "pip" # See documentation for possible values
     directory: "/"
     target-branch: "dev"
     schedule:
-      interval: "monthly"
-    groups:
-      pip:
-        patterns:
-          - "*"
-  - package-ecosystem: "npm"
-    directory: "/etc"
-    target-branch: "dev"
-    schedule:
-      interval: "monthly"
-    groups:
-      npm:
-        patterns:
-          - "*"
+      interval: "weekly"

.github/lychee.toml

@@ -1,38 +0,0 @@
-# Documentation destinations only: do not crawl archived URLs, app routes, or image assets.
-# Manual run from the repository root with Lychee 0.24.2:
-# git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > /tmp/archivebox-link-inputs.txt
-# lychee --config .github/lychee.toml --root-dir "$PWD" --files-from /tmp/archivebox-link-inputs.txt
-# Explicit tracked inputs include README symlinks and let errors in one file leave others checkable.
-include_fragments = "anchor-only"
-include_verbatim = true
-max_retries = 0
-max_concurrency = 4
-timeout = 20
-no_progress = true
-hidden = true
-extensions = ["md", "html", "py", "rst", "yml", "yaml", "sh", "toml"]
-exclude_path = ['(^|/)(tests|data|apidocs|node_modules|\.git)(/|$)']
-# Includes override this exclude; this is a documentation-link allowlist.
-exclude = ['.*']
-include = [
- '^https://github\.com/ArchiveBox/ArchiveBox/blob/[^/]+/.*\.md(?:#|$)',
- '^https://raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/',
- '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki(?:/|#|$)|/?#)',
- '^https?://docs\.archivebox\.io(?:/|$)',
- '^file:.*\.md(?:#|$)',
- '^file:.*/docs/[^/.#]+(?:#|$)',
-]
-fallback_extensions = ["md"]
-remap = [
- # Lychee's Markdown slugger drops variation selectors. Check these against rendered HTML.
-
- '^file:.*/README\.md(#%EF%B8%8F-cli-usage)$ https://github.com/ArchiveBox/ArchiveBox${1}',
- '^file:.*/docs/Install\.md(#option-a-docker--docker-compose-setup-%EF%B8%8F)$ https://github.com/ArchiveBox/ArchiveBox/wiki/Install${1}',
- # The wiki accepts this case alias, but raw file paths are case-sensitive.
- '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/Setting-up-Storage$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/Setting-Up-Storage.md',
-
- # Bare missing wiki pages silently redirect to Home; raw pages return an actual 404.
- # Keep fragment URLs on the rendered wiki to catch published heading drift.
- '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/([^#/?]+)$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/${1}.md',
-]
-preprocess = { command = "bin/lychee_preprocess.py" }

.github/pages/README.md

@@ -1,119 +0,0 @@
-# archivebox.io website
-
-This repository owns its complete website. Build and deploy it without checking
-out another repository's website, fetching shared snippets, or calling a shared
-workflow. Common files are ordinary committed copies; update them deliberately.
-
-## Repository-specific files
-
-- Content: `publicsite/index.html`.
-- Site/gallery generation: `bin/generate_ui_screenshot_gallery.py`.
-- Application capture: `bin/collect_ui_screenshots.sh`.
-- Deployment branch: `dev`.
-
-## Common publication contract
-
-1. Build pull requests without deploying them.
-2. Build website pushes immediately, independently of application capture CI.
-3. Restore validated screenshots from trusted default-branch capture jobs.
-   Keep the original capture revision, timestamp, coverage and image checksums.
-4. Retain `site-screenshots` artifacts for 90 days. After expiry, restore the
-   published image set. Never replace a complete gallery with an empty one when
-   restoration fails. Apple device groups are independently validated artifacts;
-   Android's checked-in bootstrap captures retain their local provenance.
-5. Render current content and layout around those captures. Site revision and
-   capture revision are separate facts; reusing captures must not relabel them.
-6. Serialize production restore/build/deploy under one concurrency group and
-   check out the latest default-branch revision when each build starts. Keep the
-   built revision in provenance; later release-bot commits must not block deploys.
-7. Keep platform-specific build, installation, capture, and completeness checks
-   in the application's existing capture workflow. Site-only edits do not need
-   to rerun those jobs.
-
-## Common presentation contract
-
-The base header, Apps menu and icons, footer, brand styles and navigation behavior
-should remain identical copies across the seven product sites. Put this site's
-name, navigation, CTA and extra footer resources outside the common base files.
-Preserve real links, anchor IDs, no-JavaScript navigation, keyboard operation,
-mobile layout, reduced-motion preferences and existing product-specific content.
-Screenshots stay in English. Translation content, when added, lives in separate
-HTML files; no runtime translation service or cross-repository dependency.
-
-## Editing and building
-
-`base/` contains identical independent copies across the seven product sites.
-Common Python scripts share the same behavior; repository formatters may change
-line wrapping without changing their syntax trees. The base owns the shared logo, Apps menu/icons, footer directory, navigation
-behavior and styles. `site.json` and `nav.html` contain this site's name, routes,
-links and CTA. Product content and additional footer resources stay in the native
-HTML/Jinja/Liquid templates listed above. The `ARCHIVEBOX:*` comments are build
-slots; the renderer fills them with static HTML, never browser-side fetches.
-
-The existing builder calls `site.py render` after generating its content. This
-copies `base/chrome.css` and `base/chrome.js` to `site-base/`, assembles the common
-HTML, validates local resources and writes the site revision to root `build.json`.
-Screenshot manifests are left untouched. No build reads another repository.
-
-Common changes are manual copy/paste edits. Compare `base/` and `site.py` with
-another product repository when updating them; do not add a synchronization job,
-package dependency or reusable cross-repository workflow. `nav.html`, `site.json`
-and the marked workflow build section are intentionally repository-specific.
-
-Build: `uv run --no-project python .github/pages/site.py render _site --source publicsite` after restoring screenshots and refreshing the gallery.
-
-Check the built site with real Chromium (also run by Pages CI):
-
-```sh
-uv run --no-config --no-project --with playwright==1.63.0 playwright install chromium
-uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py SITE_OUTPUT --evidence /tmp/site-evidence
-```
-
-Use this workflow's `SITE_OUTPUT` directory. The check covers desktop/mobile
-layout, keyboard dismissal, no-JavaScript links, local resources and visible
-images, and uploads its screenshots as `site-verification` for 14 days.
-
-## Branding and future translations
-
-`base/chrome.css` owns the shared system font stacks, plum accent and CTA shape,
-including the existing `.button` and `.ab-btn` classes. Product styles own their
-layouts, feature sections and platform-specific button states. Shared system
-fonts need no external font downloads. `site.py` emits the same complete SEO/OG/
-Twitter tags everywhere while retaining each native template's title, description,
-canonical URL, locale and social artwork; edit those values with that page's copy.
-
-For translations, keep complete separate HTML files under `es/`, `fr/` and `zh/`,
-and list their output paths in `site.json` alongside the English pages. Share CSS,
-JS, social artwork and the English screenshot gallery. Translate each page's title
-and description, set its own canonical and `lang` (`zh` for Chinese), and add
-reciprocal `hreflang` links only for translations that actually exist. The renderer
-preserves these tags and computes shared asset paths for nested pages. Localized
-header/footer copy and the small language selector can be added with those pages;
-there is no language redirect, placeholder translation or translation runtime now.
-
-## Screenshot strips
-
-`marquee.py` and `base/marquee.css` / `base/marquee.js` are also identical copies.
-Product homepages choose their placement with an `ARCHIVEBOX:MARQUEE` comment.
-The renderer takes real images, captions, dimensions and links from the generated
-`screenshots/index.html`; there is no second screenshot list or browser fetch.
-Desktop/tablet/mobile galleries contribute their desktop views. Apple uses
-`MARQUEE-CLIENT` and `MARQUEE-SERVER` slots to keep its two products separate.
-Catalog sites without a slot do not load the strip assets.
-
-The strip scrolls once, pauses on hover, focus or manual interaction, and has an
-explicit Play/Pause button. Reduced-motion users start paused. Without JavaScript,
-normal image links and horizontal scrolling remain available. Screenshot capture
-manifests and provenance are never rewritten by this presentation step.
-
-Standalone one- or two-image illustrations remain in their authored source.
-Gallery strips with three or more images use the latest restored captures.
-
-## Keeping the common layout in sync
-
-The comments in `base/header.html`, `base/footer-start.html`,
-`base/footer-end.html`, `base/chrome.css`, and `base/chrome.js` list the peer
-repositories. Update those committed copies together when shared branding,
-Apps links, keyboard behavior, or footer resources change. Preserve each site's
-local navigation and content. The former monorepo `site-chrome` generator is
-retired; builds must use only this repository's files.

.github/pages/artifacts.py

@@ -1,102 +0,0 @@
-"""Common screenshot artifact transport; coverage checks belong to each app."""
-
-import json
-import subprocess
-from pathlib import PurePosixPath
-from urllib.request import urlopen
-
-
-def api(repo, path):
-    return json.loads(subprocess.check_output(["gh", "api", f"repos/{repo}/{path}"]))
-
-
-def trusted(run, repo, workflow, branch, successful=True):
-    return (
-        (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
-        and run["head_branch"] == branch
-        and run["event"] in ("push", "workflow_dispatch", "schedule")
-        and run["path"] == f".github/workflows/{workflow}"
-        and (not successful or run["conclusion"] == "success")
-    )
-
-
-def runs(repo, workflow, branch, successful=True, artifact_names=("site-screenshots",)):
-    # Ask for capture artifacts directly instead of checking every historical
-    # workflow run (most app runs never produce a complete screenshot set).
-    # Artifact creation time also includes fresh captures from old rerun jobs.
-    candidates = []
-    for name in artifact_names:
-        rows = subprocess.check_output(
-            [
-                "gh",
-                "api",
-                f"repos/{repo}/actions/artifacts?name={name}&per_page=100",
-                "--paginate",
-                "--jq",
-                ".artifacts[] | select(.expired == false) | @json",
-            ],
-            text=True,
-        )
-        candidates.extend(json.loads(row) for row in rows.splitlines())
-    seen = set()
-    for artifact in sorted(candidates, key=lambda item: item["created_at"], reverse=True):
-        source = artifact.get("workflow_run") or {}
-        run_id = source.get("id")
-        if (
-            artifact["name"] not in artifact_names
-            or not run_id
-            or run_id in seen
-            or source.get("head_branch") != branch
-            or source.get("head_repository_id") != source.get("repository_id")
-        ):
-            continue
-        seen.add(run_id)
-        run = api(repo, f"actions/runs/{run_id}")
-        if trusted(run, repo, workflow, branch, successful):
-            yield run
-
-
-def names(repo, run):
-    rows = subprocess.check_output(
-        [
-            "gh",
-            "api",
-            f"repos/{repo}/actions/runs/{run['id']}/artifacts?per_page=100",
-            "--paginate",
-            "--jq",
-            ".artifacts[] | @json",
-        ],
-        text=True,
-    )
-    return {artifact["name"] for row in rows.splitlines() if not (artifact := json.loads(row))["expired"]}
-
-
-def download(repo, run, name, destination):
-    subprocess.run(
-        [
-            "gh",
-            "run",
-            "download",
-            str(run["id"]),
-            "--repo",
-            repo,
-            "--name",
-            name,
-            "--dir",
-            str(destination),
-        ],
-        check=True,
-    )
-
-
-def relative_path(name):
-    path = PurePosixPath(name)
-    if not name or path.is_absolute() or ".." in path.parts or any(c in name for c in ":\\?#%"):
-        raise ValueError(f"Unsafe screenshot path: {name}")
-    return path
-
-
-def fetch(base, name):
-    relative_path(name)
-    with urlopen(base + name, timeout=60) as response:
-        return response.read()

.github/pages/base/chrome.css

@@ -1,156 +0,0 @@
-/*
-Shared ArchiveBox website navigation styles and behavior.
-Keep this file in sync with .github/pages/base/chrome.css in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
-*/
-/* Common brand tokens. Native templates own product layout and content. */
-:root {
-  --abx-font-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
-  --abx-font-mono: ui-monospace, "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace;
-  --abx-accent: #9b2854;
-  --abx-accent-hover: #782040;
-  --abx-paper: #fcfaf7;
-  --abx-ink: #28252a;
-  --abx-muted: #716c74;
-  --abx-line: #e9e2e3;
-  --font-sans: var(--abx-font-sans);
-  --font-mono: var(--abx-font-mono);
-  --ab-font-sans: var(--abx-font-sans);
-  --ab-font-serif: var(--abx-font-sans);
-  --ab-font-label: var(--abx-font-sans);
-  --ab-font-mono: var(--abx-font-mono);
-  --ab-accent: var(--abx-accent);
-  --ab-accent-hover: var(--abx-accent-hover);
-  --ab-on-accent: #fff;
-}
-body { font-family: var(--abx-font-sans); }
-code, pre, kbd, samp { font-family: var(--abx-font-mono); }
-/* Reuse existing CTA classes; platform selection continues to control colors. */
-.button, .ab-btn {
-  display: inline-flex; align-items: center; justify-content: center; gap: 10px;
-  min-height: 44px; height: auto; max-width: 100%; padding: 10px 20px;
-  border-radius: 24px; font-family: var(--abx-font-sans);
-  font-size: 14px; font-weight: 650; line-height: 1.4;
-  text-align: center; white-space: normal; text-decoration: none;
-}
-.button.primary, .ab-btn--primary { background: var(--abx-accent); border-color: var(--abx-accent); color: #fff; }
-.button.primary:hover, .ab-btn--primary:hover { background: var(--abx-accent-hover); border-color: var(--abx-accent-hover); color: #fff; }
-.button:focus-visible, .ab-btn:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
-.button svg, .button img, .ab-btn svg, .ab-btn img { flex-shrink: 0; }
-@media (prefers-reduced-motion: reduce) { .button, .ab-btn { transition: none; } .button:hover { transform: none; } }
-
-/* Shared ArchiveBox site chrome. Independent copy; edit alongside the other product sites, without a build dependency. */
-/* Native anchor navigation leaves room for the heading and its surrounding content. */
-html { scroll-behavior: smooth; scroll-padding-top: clamp(96px, 25vh, 240px); }
-@media (prefers-reduced-motion: reduce) { html { scroll-behavior: auto; } }
-.abx-header, .abx-footer { box-sizing: border-box; font-family: var(--abx-font-sans); font-size: 14px; line-height: 1.5; letter-spacing: normal; text-align: left; }
-.abx-header *, .abx-footer * { box-sizing: border-box; }
-.abx-header { position: sticky; top: 0; z-index: 100; display: flex; align-items: center; justify-content: space-between; gap: 24px; width: 100%; max-width: none; margin: 0; padding: 15px max(24px, calc((100% - 1280px) / 2)); border: 0; border-bottom: 1px solid #e9e2e3; background: #fcfaf7f5; color: #28252a; backdrop-filter: blur(16px); }
-.abx-header a, .abx-footer a { text-decoration: none; text-underline-offset: .24em; }
-.abx-header .abx-brand, .abx-footer .abx-brand { display: inline-flex; flex: 0 0 auto; align-items: center; gap: 11px; margin: 0; padding: 0; color: inherit; font-size: 19px; font-weight: 750; line-height: 1.2; letter-spacing: -.5px; white-space: nowrap; }
-.abx-brand .abx-logo { display: block; flex: 0 0 auto; width: 32px; height: 32px; margin: 0; }
-.abx-header .abx-subsite { transform: translateY(3px); margin-left: 2px; color: #716c74; font-size: 12px; font-weight: 450; letter-spacing: 0; }
-.abx-header .abx-nav { display: flex; align-items: center; justify-content: flex-end; flex-wrap: wrap; gap: 0; width: auto; max-width: none; margin: 0; padding: 0; font-size: 13px; font-weight: 600; }
-.abx-nav > a, .abx-nav > details { margin: 0 0 0 15px; padding: 0 0 0 15px; border: 0; border-left: 1px solid #ded4d8; border-radius: 0; background: none; }
-.abx-nav > :first-child { margin-left: 0; padding-left: 0; border-left: 0; }
-.abx-header .abx-nav a { display: inline-flex; align-items: center; min-height: 36px; color: #4d424b; white-space: nowrap; }
-.abx-header .abx-nav a:hover { color: #9b2854; }
-.abx-header .abx-nav .abx-cta { margin-left: 18px; padding: 9px 17px; border: 1px solid #9b2854; border-radius: 24px; background: #9b2854; color: #fff; }
-.abx-header .abx-nav .abx-cta:hover { background: #782040; color: #fff; }
-.abx-header .abx-apps { position: relative; }
-.abx-apps > summary { display: flex; align-items: center; gap: 7px; min-height: 36px; list-style: none; cursor: pointer; color: #4d424b; }
-.abx-apps > summary::-webkit-details-marker { display: none; }
-.abx-apps > summary::before { content: '›'; display: inline-block; color: #9b2854; font-size: 22px; line-height: 1; transition: transform .15s; }
-.abx-apps[open] > summary::before { transform: rotate(90deg); }
-.abx-header .abx-app-links { position: absolute; right: 0; top: calc(100% + 12px); width: 440px; max-width: calc(100vw - 24px); max-height: min(70vh, 580px); overflow-y: auto; padding: 8px; border: 1px solid #e9e2e3; border-radius: 16px; background: #fcfaf7; box-shadow: 0 18px 48px #2d152522, 0 3px 8px #2d152508; }
-.abx-header .abx-app-row { display: flex; align-items: center; gap: 8px; min-height: 44px; padding: 4px 6px; border-radius: 10px; transition: background .15s; }
-.abx-header .abx-app-row:hover, .abx-header .abx-app-row:focus-within { background: #f2e8ed; }
-.abx-header .abx-app-links .abx-app-name { flex: 1; min-width: 0; min-height: 34px; padding: 5px 4px; border-radius: 6px; font-size: 13px; line-height: 1.35; white-space: normal; }
-.abx-header .abx-platforms { display: flex; flex: 0 0 auto; align-items: center; gap: 5px; }
-.abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 auto; height: 30px; min-height: 30px; padding: 5px; border: 1px solid #e8dfe3; border-radius: 9px; background: #fffdfb; color: #4d424b; box-shadow: 0 2px 3px #2d152510, inset 0 1px 0 #fff; transition: transform .15s, box-shadow .15s, border-color .15s; }
-.abx-header .abx-app-links .abx-platform { width: 30px; }
-.abx-header .abx-platform svg { display: block; width: 18px; height: 18px; margin: 0; fill: currentColor; }
-.abx-header .abx-app-links .abx-platform-docker { color: #2386ce; }
-.abx-header .abx-app-links .abx-platform-windows, .abx-header .abx-app-links .abx-platform-app-store, .abx-header .abx-app-links .abx-platform-safari { color: #1879c4; }
-.abx-header .abx-app-links .abx-platform-android { color: #29965d; }
-.abx-header .abx-app-links .abx-platform-firefox { color: #e46b24; }
-.abx-header .abx-app-links .abx-platform-brave { color: #e8552b; }
-.abx-header .abx-app-links .abx-platform-edge { color: #168b95; }
-.abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-platform:focus-visible, .abx-header .abx-app-links .abx-tool:hover, .abx-header .abx-app-links .abx-tool:focus-visible { transform: translateY(-2px); border-color: #c99bb0; background: #fff; box-shadow: 0 5px 10px #2d15251c; }
-.abx-header .abx-app-links .abx-tool { padding: 5px 8px; font-size: 11px; white-space: nowrap; }
-.abx-header .abx-tools-row { margin-top: 5px; padding-top: 9px; border-top: 1px solid #e9e2e3; border-radius: 0 0 10px 10px; }
-.abx-header .abx-tools-row .abx-app-name { flex: 0 0 auto; }
-.abx-header .abx-tools-row .abx-platforms { flex: 0 1 auto; flex-wrap: wrap; justify-content: flex-end; margin-left: auto; }
-.abx-header .abx-app-links code { display: inline; vertical-align: baseline; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 1em; font-weight: inherit; line-height: inherit; margin: 0; padding: 0; border: 0; background: transparent; color: inherit; }
-.abx-header .abx-icon-stack { isolation: isolate; gap: 0; padding: 4px 5px; }
-.abx-header .abx-icon-stack .abx-integration { position: relative; z-index: 1; transform: translateY(var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); transition: transform .18s ease, box-shadow .18s ease, border-color .18s ease; }
-.abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -10px; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(2) { --stack-angle: 9deg; --stack-y: -2px; z-index: 2; color: #569a31; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(3) { --stack-angle: -6deg; --stack-y: 2px; z-index: 3; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(4) { --stack-angle: 7deg; --stack-y: -1px; z-index: 4; color: #f00; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(5) { --stack-angle: -10deg; --stack-y: 1px; z-index: 5; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(6) { --stack-angle: 6deg; --stack-y: -2px; z-index: 6; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(7) { --stack-angle: -5deg; z-index: 7; }
-.abx-header .abx-icon-stack .abx-integration:not(:hover, :focus-visible):has(~ .abx-integration:is(:hover, :focus-visible)) { transform: translate(-6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
-.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) ~ .abx-integration:not(:hover, :focus-visible) { transform: translate(6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
-.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { z-index: 20; transform: translateY(-7px) rotate(0deg) scale(1.12); border-color: #c99bb0; box-shadow: 0 8px 14px #2d15252b; }
-@media (hover: none) { .abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -6px; } }
-@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration { transition: none; } }
-.abx-footer { position: relative; z-index: 1; display: block; width: 100%; max-width: none; margin: 72px 0 0; padding: 60px max(24px, calc((100% - 1280px) / 2)) 24px; border: 0; border-top: 3px solid #7e284a; background: radial-gradient(ellipse at 12% 0, #54243b55, transparent 48%), linear-gradient(120deg, #201923, #14151b 75%); color: #ddd2dd; color-scheme: dark; }
-.abx-footer .abx-footer-intro { display: flex; align-items: flex-start; justify-content: space-between; gap: 32px; margin: 0 0 42px; }
-.abx-footer .abx-brand { color: #fff4ed; font-size: 24px; }
-.abx-footer .abx-brand .abx-logo { width: 40px; height: 40px; }
-.abx-footer .abx-tagline { max-width: 430px; margin: 12px 0 0; color: #b8acbb; font-size: 14px; line-height: 1.7; }
-.abx-footer .abx-footer-motto { margin: 2px 0 0; color: #d8b39f; font: italic 25px/1.45 Georgia, serif; text-align: right; }
-.abx-footer .abx-footer-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 42px; margin: 0; }
-.abx-footer .abx-footer-column { margin: 0; padding: 20px 0 0; border: 0; border-top: 1px solid #67505e66; background: none; }
-.abx-footer .abx-footer-column h2 { margin: 0 0 19px; padding: 0; border: 0; color: #e4acbb; font: 650 12px/1.4 var(--abx-font-sans); letter-spacing: .12em; text-transform: uppercase; }
-.abx-footer .abx-footer-column ul { display: block; list-style: none; margin: 0; padding: 0; }
-.abx-footer .abx-footer-column li { display: flex; align-items: center; gap: 10px; margin: 0; padding: 0; min-height: 36px; }
-.abx-footer .abx-footer-column li::before { content: none; }
-.abx-footer .abx-footer-column a { color: #ddd2dd; font-size: 13px; line-height: 1.55; }
-.abx-footer .abx-footer-column a:first-child { padding: 6px 0; }
-.abx-footer a:hover { color: #fff; text-decoration: underline; }
-.abx-footer .abx-source { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 28px; width: 28px; height: 28px; border: 1px solid transparent; border-radius: 50%; color: #a58d9f; }
-.abx-footer .abx-source:hover { color: #f3b5cb; border-color: #80516a; background: #ffffff08; }
-.abx-source svg { width: 15px; height: 15px; fill: currentColor; }
-.abx-footer .abx-footer-bottom { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px 24px; margin-top: 32px; padding-top: 21px; border-top: 1px solid #67505e66; color: #a99bab; font-size: 12px; }
-.abx-footer .abx-footer-bottom p { margin: 0; font-size: inherit; color: inherit; }
-.abx-footer .abx-footer-bottom a { color: #cebbc9; }
-.abx-footer .abx-footer-local { margin: 24px 0 0; padding: 16px 0 0; border: 0; border-top: 1px solid #67505e66; border-radius: 0; background: none; color: #baacbc; font-size: 12px; }
-.abx-footer-local > summary { cursor: pointer; color: #d6bfce; }
-.abx-footer-local > div { padding-top: 18px; }
-.abx-footer-local a { color: #e5aabd; }
-.abx-footer-local .brand, .abx-footer-local .footer-intro, .abx-footer-local .footer-grid, .abx-footer-local .footer-meta { color: inherit; }
-.abx-footer-local .footer-card { background: transparent; border-color: #67505e66; }
-.abx-footer-local h2, .abx-footer-local h3, .abx-footer-local strong { color: #e4d8e3; }
-.abx-footer-local .footer-link-copy span, .abx-footer-local .footer-card-subtitle { color: #baacbc; }
-.abx-header a:focus-visible, .abx-header summary:focus-visible, .abx-footer a:focus-visible, .abx-footer summary:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
-@media (max-width: 1100px) {
-  .abx-header { flex-wrap: wrap; gap: 8px 24px; padding: 12px 20px; }
-  .abx-header .abx-nav { justify-content: flex-start; }
-  .abx-header .abx-app-links { position: fixed; top: auto; left: 20px; right: auto; width: 440px; margin-top: 10px; }
-}
-@media (max-width: 700px) {
-  .abx-header { position: relative; padding: 14px 18px; }
-  .abx-header .abx-brand { flex-wrap: wrap; row-gap: 5px; }
-  .abx-header .abx-nav { width: 100%; gap: 4px 0; font-size: 12px; }
-  .abx-nav > a, .abx-nav > details { margin-left: 10px; padding-left: 10px; }
-  .abx-header .abx-nav .abx-cta { margin-left: 10px; padding: 7px 13px; }
-  .abx-header .abx-app-links { left: 12px; right: 12px; width: auto; max-height: 60vh; }
-  .abx-footer { padding: 40px 22px 22px; margin-top: 48px; }
-  .abx-footer .abx-footer-intro { display: block; margin-bottom: 30px; }
-  .abx-footer .abx-footer-motto { margin-top: 20px; text-align: left; font-size: 22px; }
-  .abx-footer .abx-footer-grid { grid-template-columns: 1fr; gap: 26px; }
-  .abx-footer .abx-footer-column li { min-height: 40px; }
-}
-@media (prefers-reduced-motion: reduce) { .abx-apps > summary::before, .abx-header .abx-app-row, .abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { transition: none; } .abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-tool:hover { transform: none; } }
-
-.abx-footer-local > div > a + a { margin-left: 12px; }
-
-@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { transform: none; } }

.github/pages/base/chrome.js

@@ -1,28 +0,0 @@
-/*
-Shared ArchiveBox website navigation styles and behavior.
-Keep this file in sync with .github/pages/base/chrome.js in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
-*/
-(() => {
-  const menu = document.querySelector('.abx-apps');
-  document.addEventListener('keydown', event => {
-    if (event.key === 'Escape' && menu?.open) { menu.open = false; menu.querySelector('summary').focus(); }
-  });
-  document.addEventListener('click', event => { if (menu?.open && !menu.contains(event.target)) menu.open = false; });
-  function revealAnchor() {
-    let target;
-    try { target = document.getElementById(decodeURIComponent(window.location.hash.slice(1))); } catch { return; }
-    if (!target) return;
-    let details = target.closest('details');
-    let opened = false;
-    while (details) { if (!details.open) { details.open = true; opened = true; } details = details.parentElement.closest('details'); }
-    if (opened) window.requestAnimationFrame(() => target.scrollIntoView());
-  }
-  window.addEventListener('hashchange', revealAnchor);
-  revealAnchor();
-})();

.github/pages/base/footer-end.html

@@ -1,12 +0,0 @@
-<!--
-Shared ArchiveBox website footer.
-Keep this file in sync with .github/pages/base/footer-end.html in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
--->
-  <div class="abx-footer-bottom"><p>Built in the open. Supported by our community.</p><p><a href="https://github.com/ArchiveBox">GitHub organization</a> · <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Security &amp; privacy</a> · <a href="https://archivebox.io">ArchiveBox.io ↗</a></p></div>
-</footer>

.github/pages/base/footer-start.html

@@ -1,13 +0,0 @@
-<!--
-Shared ArchiveBox website footer.
-Keep this file in sync with .github/pages/base/footer-start.html in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
--->
-<footer class="abx-footer" aria-label="ArchiveBox ecosystem">
-  <div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
-  <div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated

.github/pages/base/header.html

@@ -1,24 +0,0 @@
-<!--
-Shared ArchiveBox website navbar and Apps menu.
-Keep this file in sync with .github/pages/base/header.html in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
--->
-<header class="abx-header">
-  <a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span><span class="abx-subsite">__SITE_NAME__</span></a>
-  <nav class="abx-nav" aria-label="Main navigation">
-    <details class="abx-apps"><summary>Apps</summary><div class="abx-app-links">
-      <div class="abx-app-row"><a class="abx-app-name" href="https://archivebox.io/#install-compose">ArchiveBox Server</a><div class="abx-platforms"><a class="abx-platform abx-platform-docker" href="https://archivebox.io/#install-compose" aria-label="ArchiveBox Server for Docker" title="ArchiveBox Server for Docker"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M349.9 236.3l-66.1 0 0-59.4 66.1 0 0 59.4zm0-204.3l-66.1 0 0 60.7 66.1 0 0-60.7zm78.2 144.8l-66.1 0 0 59.4 66.1 0 0-59.4zM271.8 104.7l-66.1 0 0 60.1 66.1 0 0-60.1zm78.1 0l-66.1 0 0 60.1 66.1 0 0-60.1zm276.8 100c-14.4-9.7-47.6-13.2-73.1-8.4-3.3-24-16.7-44.9-41.1-63.7l-14-9.3-9.3 14c-18.4 27.8-23.4 73.6-3.7 103.8-8.7 4.7-25.8 11.1-48.4 10.7l-434.7 0c-8.7 50.8 5.8 116.8 44 162.1 37.1 43.9 92.7 66.2 165.4 66.2 157.4 0 273.9-72.5 328.4-204.2 21.4 .4 67.6 .1 91.3-45.2 1.5-2.5 6.6-13.2 8.5-17.1l-13.3-8.9zM115.6 176.8l-66 0 0 59.4 66.1 0 0-59.4-.1 0zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm-78.1-72.1l-66.1 0 0 60.1 66.1 0 0-60.1z"/></svg></a><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/#archivebox-serverapp" aria-label="ArchiveBox Server for macOS" title="ArchiveBox Server for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://electron.archivebox.io/" aria-label="ArchiveBox Server for Windows" title="ArchiveBox Server for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://archivebox.io/#install-apt" aria-label="ArchiveBox Server for Linux" title="ArchiveBox Server for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-1Line truncated
-      <div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Desktop</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Desktop for macOS" title="ArchiveBox Desktop for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://windows.archivebox.io/" aria-label="ArchiveBox Desktop for Windows" title="ArchiveBox Desktop for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://linux.archivebox.io/" aria-label="ArchiveBox Desktop for Linux" title="ArchiveBox Desktop for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-10 14.1-10.5 4.7-4.4 13.5-14.2 27.9-14.2 7.1 0 15.6 2.3 25.9 8.9 6.3 4.1 11.3 4.4 22.6 9.3 8.4 3.5 13.7 9.7 10.5 18.2-2.6 7.1-11 14.4-22.7 18.1-11.1 3.6-19.8 16-38.2 14.9-3.9-.2-7-1-9.6-2.1-8-3.5-12.2-10.4-20-15-8.6-4.8-13.2-10.4-14.7-15.3-1.4-4.9 0-9 4.2-12.3zm3.3 334c-2.7 35.1-43.9 34.4-75.3 18-29.9-15.8-68.6-6.5-76.5-21.9-2.4-4.7-2.4-12.7 2.6-26.4l0-.2c2.4-7.6 .6-16-.6-23.9-1.2-7.8-1.8-15 .9-20 3.5-6.7 8.5-9.1 14.8-11.3 10.3-3.7 11.8-3.4 19.6-9.9 5.5-5.7 9.5-12.9 14.3-18 5.1-5.5 10-8.1 17.7-6.9 8.1 1.2 15.1 6.8 21.9 16l19.6 35.6c9.5 19.9 43.1 48.4 41 68.9zm-1.4-25.9c-4.1-6.6-9.6-13.6-14.4-19.6 7.1 0 14.2-2.2 16.7-8.9 2.3-6.2 0-14.9-7.4-24.9-13.5-18.2-38.3-32.5-38.3-32.5-13.5-8.4-21.1-18.7-24.6-29.9s-3-23.3-.3-35.2c5.2-22.9 18.6-45.2 27.2-59.2 2.3-1.7 .8 3.2-8.7 20.8-8.5 16.1-24.4 53.3-2.6 82.4 .6-20.7 5.5-41.8 13.8-61.5 12-27.4 37.3-74.9 39.3-112.7 1.1 .8 4.6 3.2 6.2 4.1 4.6 2.7 8.1 6.7 12.6 10.3 12.4 10 28.5 9.2 42.4 1.2 6.2-3.5 11.2-7.5 15.9-9 9.9-3.1 17.8-8.6 22.3-15 7.7 30.4 25.7 74.3 37.2 95.7 6.1 11.4 18.3 35.5 23.6 64.6 3.3-.1 7 .4 10.9 1.4 13.8-35.7-11.7-74.2-23.3-84.9-4.7-4.6-4.9-6.6-2.6-6.5 12.6 11.2 29.2 33.7 35.2 59 2.8 11.6 3.3 23.7 .4 35.7 16.4 6.8 35.9 17.9 30.7 34.8-2.2-.1-3.2 0-4Line truncated
-      <div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Mobile</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Mobile for iOS" title="ArchiveBox Mobile for iOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-android" href="https://android.archivebox.io/" aria-label="ArchiveBox Mobile for Android" title="ArchiveBox Mobile for Android"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!-- Font Awesome Free 5.0.13 by Fonticons, Inc. https://fontawesome.com License: https://creativecommons.org/licenses/by/4.0/ --><path d="M89.6 204.5v115.8c0 15.4-12.1 27.7-27.5 27.7-15.3 0-30.1-12.4-30.1-27.7V204.5c0-15.1 14.8-27.5 30.1-27.5 15.1 0 27.5 12.4 27.5 27.5zm10.8 157c0 16.4 13.2 29.6 29.6 29.6h19.9l.3 61.1c0 36.9 55.2 36.6 55.2 0v-61.1h37.2v61.1c0 36.7 55.5 36.8 55.5 0v-61.1h20.2c16.2 0 29.4-13.2 29.4-29.6V182.1H100.4v179.4zm248-189.1H99.3c0-42.8 25.6-80 63.6-99.4l-19.1-35.3c-2.8-4.9 4.3-8 6.7-3.8l19.4 35.6c34.9-15.5 75-14.7 108.3 0L297.5 34c2.5-4.3 9.5-1.1 6.7 3.8L285.1 73c37.7 19.4 63.3 56.6 63.3 99.4zm-170.7-55.5c0-5.7-4.6-10.5-10.5-10.5-5.7 0-10.2 4.8-10.2 10.5s4.6 10.5 10.2 10.5c5.9 0 10.5-4.8 10.5-10.5zm113.4 0c0-5.7-4.6-10.5-10.2-10.5-5.9 0-10.5 4.8-10.5 10.5s4.6 10.5 10.5 10.5c5.6 0 10.2-4.8 10.2-10.5zm94.8 60.1c-15.1 0-27.5 12.1-27.5 27.5v115.8c0 15.4 12.4 27.7 27.5 27.7 15.4 0 30.1-12.4 30.1-27.7V204.5c0-15.4-14.8-27.5-30.1-27.5z"/></svg></a></div></div>
-      <div class="abx-app-row"><a class="abx-app-name" href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><div class="abx-platforms"><a class="abx-platform abx-platform-chrome" href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj" aria-label="ArchiveBox Browser Extension for Chrome" title="ArchiveBox Browser Extension for Chrome"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc. --><path d="M0 256C0 209.4 12.47 165.6 34.27 127.1L144.1 318.3C166 357.5 207.9 384 256 384C270.3 384 283.1 381.7 296.8 377.4L220.5 509.6C95.9 492.3 0 385.3 0 256zM365.1 321.6C377.4 302.4 384 279.1 384 256C384 217.8 367.2 183.5 340.7 160H493.4C505.4 189.6 512 222.1 512 256C512 397.4 397.4 511.1 256 512L365.1 321.6zM477.8 128H256C193.1 128 142.3 172.1 130.5 230.7L54.19 98.47C101 38.53 174 0 256 0C350.8 0 433.5 51.48 477.8 128V128zM168 256C168 207.4 207.4 168 256 168C304.6 168 344 207.4 344 256C344 304.6 304.6 344 256 344C207.4 344 168 304.6 168 256z"/></svg></a><a class="abx-platform abx-platform-safari" href="https://extension.archivebox.io/" aria-label="ArchiveBox Browser Extension for Safari" title="ArchiveBox Browser Extension for Safari"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M274.7 274.7L237.3 237.3 166 346 274.7 274.7zM256 8a248 248 0 1 0 0 496 248 248 0 1 0 0-496zM411.8 182.8l14.8-6.1c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.8 1 2.6 1.7 1.3 1.6 1.7 2.6 .6 2 .6 3.1-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.3-2.6 1.7L418 197.6c-2 .8-4.2 .8-6.1 0s-3.5-2.4-4.3-4.3-.8-4.2 0-6.1 2.4-3.5 4.3-4.3zM314.4 94l6.1-14.8c.8-2 2.4-3.5 4.3-4.3s4.2-.8 6.1 0 3.5 2.4 4.3 4.3 .8 4.2 0 6.1l-6.1 14.8c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0c-1-.4-1.9-1-2.6-1.7s-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1zM256 60c2.1 0 4.2 .8 5.7 2.3S264 65.9 264 68l0 16c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7S253.9 60 256 60zM181 74.9c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.9 1 2.6 1.7 1.3 1.6 1.7 2.6L197.6 94c.4 1 .7 2 .7 3.1s-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.4-2.6 1.8-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.8-1.3-1.7-1.7-2.7l-6.1-14.8c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3zm-63.6 42.5c1.5-1.5 3.5-2.3 5.7-2.3s4.2 .8 5.7 2.3L140 128.7c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3l-11.3-11.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6l0 0zM60 256c0-2.1 .8-4.2 2.3-5.7S65.9 248 68 248l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3l-16 0c-2.1 0-4.2-.8-5.7-2.3S60 258.1 60 256zm40.2 73.2l-14.8 6.1c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1 1-1.9 1.7-2.6 1.6-1.3 2.6-1.7L94 314.4c2-.8 4.2-.8 6.1 0s3.5 2.4 4.3 4.3 .8 4.2 0 6.1-2.4 3.5-4.3 4.3zm4.3-136c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1 2.4-3.5 4.3-4.3 4.2-.8 6.1 0l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1l0 0zM197.6 418l-6.1 14.8c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1l6.1-14.8c.4-1 1-1.9 1.7-2.7s1.6-1.4 2.6-1.8 2-.6 3.1-.6 2.1 .2 3.1 .6 1.9 1 2.6 1.8 1.3 1.7 1.7 2.6 .6 2 .6 3.1-.2 2.1-.7 3.1zM264 444c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7s3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3 2.3 3.5 2.3 5.7l0 16zm67-6.9c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6L314.4 418c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3 4.2-.8 6.1 0 3.5 2.4 4.3 4.3l6.1 14.8c.8 2 .8 4.2 0 6.1s-2.4 3.5-4.3 4.3zm63.6-42.5c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3L372 383.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6 3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3l11.3 11.3c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6l0 0zM286.2 286.2L110.3 401.7 225.8 225.8 401.7 110.3 286.2 286.2zM437.1 331c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1c.4-1 1-1.9 1.7-2.6s1.6-1.3 2.6-1.7 2-.6 3.1-.6 2.1 .2 3.1 .6l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1zm6.9-67l-16 0c-2.1 0-4.2-.8-5.7-2.3s-2.3-3.5-2.3-5.7 .8-4.2 2.3-5.7 3.5-2.3 5.7-2.3l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3z"/></svg></a><a class="abx-platform abx-platform-firefox" href="https://addons.mozilla.org/firefox/addon/archivebox-exporter/" aria-label="ArchiveBox Browser Extension for Firefox" title="ArchiveBox Browser Extension for Firefox"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512Line truncated
-      <div class="abx-app-row abx-plugin-row"><a class="abx-app-name" href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><div class="abx-platforms abx-icon-stack"><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Google Drive storage" title="Google Drive storage"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="m38.7 419.3 22.6 39c4.7 8.2 11.4 14.7 19.4 19.4l80.6-139.6H0c0 9.1 2.3 18.2 7 26.4z" style="fill:#0066da"/><path d="M256 173.9 175.4 34.3c-7.9 4.7-14.7 11.1-19.4 19.4L7 311.7c-4.6 8-7 17.1-7 26.4h161.3z" style="fill:#00ac47"/><path d="M431.4 477.7c7.9-4.7 14.7-11.1 19.4-19.4l9.4-16.1 44.9-77.7c4.7-8.2 7-17.3 7-26.4H350.7l34.3 67.4z" style="fill:#ea4335"/><path d="m256 173.9 80.6-139.6c-7.9-4.7-17-7-26.4-7H201.8c-9.4 0-18.5 2.6-26.4 7z" style="fill:#00832d"/><path d="M350.7 338.1H161.3L80.6 477.7c7.9 4.7 17 7 26.4 7h298c9.4 0 18.5-2.6 26.4-7z" style="fill:#2684fc"/><path d="M430.5 182.7 356 53.7c-4.7-8.2-11.4-14.7-19.4-19.4L256 173.9l94.7 164.2h161c0-9.1-2.3-18.2-7-26.4z" style="fill:#ffba00"/></svg></a><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Amazon S3 storage" title="Amazon S3 storage"><svg aria-hidden="true" focusable="false" fill="#569A31" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M20.913 13.147l.12-.895c.947.576 1.258.922 1.354 1.071-.16.031-.562.046-1.474-.176zm-2.174 7.988a.547.547 0 0 0-.005.073c0 .084-.207.405-1.124.768a10.28 10.28 0 0 1-1.438.432c-1.405.325-3.128.504-4.853.504-4.612 0-7.412-1.184-7.412-1.704a.547.547 0 0 0-.005-.073L1.81 5.602c.135.078.28.154.432.227.042.02.086.038.128.057.134.062.272.122.417.18l.179.069c.154.058.314.114.478.168.043.013.084.029.13.043.207.065.423.127.646.187l.176.044c.175.044.353.087.534.127a23.414 23.414 0 0 0 .843.17l.121.023c.252.045.508.085.768.122.071.011.144.02.216.03.2.027.4.053.604.077l.24.027c.245.026.49.05.74.07l.081.009c.275.022.552.04.83.056l.233.012c.21.01.422.018.633.025a33.088 33.088 0 0 0 2.795-.026l.232-.011c.278-.016.555-.034.83-.056l.08-.008c.25-.02.497-.045.742-.072l.238-.026c.205-.024.408-.05.609-.077.07-.01.141-.019.211-.03.261-.037.519-.078.772-.122l.111-.02c.215-.04.427-.082.634-.125l.212-.047c.186-.041.368-.085.546-.13l.166-.042c.225-.06.444-.122.654-.189.04-.012.077-.026.115-.038a10.6 10.6 0 0 0 .493-.173c.058-.021.114-.044.17-.066.15-.06.293-.12.43-.185.038-.017.079-.034.116-.052.153-.073.3-.15.436-.228l-.976 7.245c-2.488-.78-5.805-2.292-7.311-3a1.09 1.09 0 0 0-1.088-1.085c-.6 0-1.088.489-1.088 1.088 0 .6.488 1.089 1.088 1.089.196 0 .378-.056.537-.148 1.72.812 5.144 2.367 7.715 3.15zm-7.42-20.047c5.677 0 9.676 1.759 9.75 2.736l-.014.113c-.01.033-.031.067-.048.101-.015.028-.026.057-.047.087-.024.033-.058.068-.09.102-.028.03-.051.06-.084.09-.038.035-.087.07-.133.105-.04.03-.074.06-.119.091-.053.036-.116.071-.177.107-.05.03-.095.06-.15.09-.068.036-.147.073-.222.11-.059.028-.114.057-.177.085-.084.038-.177.074-.268.111-.068.027-.13.054-.203.082-.097.036-.205.072-.31.107-.075.026-.148.053-.228.079-.111.035-.233.069-.35.103-.085.024-.165.05-.253.073-.124.034-.258.065-.389.098-.093.022-.181.046-.278.068-.139.032-.287.061-.433.091-.098.02-.191.041-.293.06-.155.03-.32.057-.482.084-.1.018-.198.036-.302.052-.166.026-.342.048-.515.072-.11.014-.213.03-.325.044-.181.023-.372.041-.56.06-.11.012-.218.025-.332.036-.188.016-.386.029-.58.043-.122.009-.24.02-.364.028-.207.012-.422.02-.635.028-.12.005-.234.012-.354.016a35.605 35.605 0 0 1-2.069 0c-.12-.004-.234-.011-.352-.016-.214-.008-.43-.016-.637-.028-.122-.008-.238-.02-.36-.027-.195-.015-.394-.028-.584-.044-.11-.01-.215-.024-.324-.035-.19-.02-.384-.038-.568-.06l-.315-.044c-.176-.024-.355-.046-.525-.073-.1-.015-.192-.033-.29-.05-.167-.028-.335-.055-.494-.086-.096-.018-.183-.038-.276-.056-.151-.032-.305-.062-.45-.095-.09-.02-.173-.043-.26-.064-.138-.034-.277-.067-.407-.102-.082-.022-.157-.046-.235-.069a11.75 11.75 0 0 1-.368-.108c-.075-.024-.141-.049-.213-.073-.11-.037-.223-.075-.325-.113-.067-.025-.125-.051-.188-.077-.096-.038-.195-.076-.282-.115-.06-.027-.11-.054-.166-.08-.08-.039-.162-.077-.233-.116-.052-.028-.094-.055-.142-.084-.063-.038-.13-.075-.185-.113-.043-.029-.075-.058-.113-.086-.048-.037-.098-.073-.139-.11-.032-.029-.054-.057-.08-.087-.033-.035-.069-.07-.093-.104-.02-.03-.031-.058-.046-.086-.018-.035-.039-.068-.049-.102l-.015-.113c.076-.977 4.074-2.736 9.748-2.736zm12.182 12.124c-.118-.628-.84-1.291-2.31-2.128l.963-7.16a.531.531 0 0 0 .005-.073C22.16 1.581 16.447 0 11.32 0 6.194 0 .482 1.581.482 3.851a.58.58 0 0 0 .005.072L2.819 21.25c.071 2.002 5.236 2.75 8.5 2.75 1.805 0 3.615-.188 5.098-.531.598-.138 1.133-.3 1.592-.48 1.18-.467 1.789-1.053 1.813-1.739l.945-7.018c.557.131 1.016.197 1.389.197.54 0 .902-.137 1.134-.413aLine truncated
-      <div class="abx-app-row abx-tools-row"><a class="abx-app-name" href="https://abx-dl.archivebox.io/">Tools</a><div class="abx-platforms"><a class="abx-tool" href="https://abx-dl.archivebox.io/"><span>One-shot <code>abx-dl</code> CLI</span></a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/tree/dev/archivebox/mcp">MCP</a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-api-usage">API</a><a class="abx-tool" href="https://web.archivebox.io">Demo</a></div></div>
-    </div></details>
-    __SITE_NAV__
-  </nav>
-</header>

.github/pages/base/marquee.css

@@ -1,11 +0,0 @@
-/* Identical standalone gallery strip across ArchiveBox product sites. */
-/* The root query container measures the usable width, excluding scrollbars. */
-html { container-type: inline-size; }
-.abx-marquee { min-width: 0; width: 100cqw; max-width: none; margin: 24px calc(50% - 50cqw) 40px; padding: 0; overflow: hidden; }
-.abx-marquee .abx-marquee-viewport { width: 100%; overflow-x: auto; scroll-behavior: auto; overscroll-behavior-x: contain; scrollbar-width: thin; padding: 16px 24px 28px; box-sizing: border-box; }
-.abx-marquee .abx-marquee-track { display: flex; align-items: flex-start; gap: 24px; width: max-content; margin-inline: auto; }
-.abx-marquee .abx-marquee-card { display: block; flex: none; width: min-content; overflow: hidden; border-radius: 10px; background: #fff; box-shadow: 0 0 0 1px rgb(0 0 0 / 12%), 0 2px 4px rgb(0 0 0 / 10%), 0 8px 16px rgb(0 0 0 / 16%); text-decoration: none; color: #533344; }
-.abx-marquee .abx-marquee-card img { display: block; height: clamp(240px, 30vw, 400px); width: auto; max-width: none; max-height: none; object-fit: contain; box-shadow: none; border: 0; border-radius: 0; background: transparent; padding: 0; margin: 0; }
-.abx-marquee :focus-visible { outline: 2px solid #9b2854; outline-offset: 3px; }
-.abx-marquee .abx-marquee-viewport:focus-visible { outline-offset: -3px; }
-@media (max-width: 600px) { .abx-marquee .abx-marquee-track { gap: 16px; } .abx-marquee .abx-marquee-card img { height: 240px; } }

.github/pages/base/marquee.js

@@ -1,44 +0,0 @@
-/* Scroll real gallery links once; keep them usable without JavaScript. */
-(() => {
-  document.querySelectorAll('.abx-marquee').forEach(gallery => {
-    const viewport = gallery.querySelector('.abx-marquee-viewport');
-    const reducedMotion = window.matchMedia('(prefers-reduced-motion: reduce)');
-    let paused = reducedMotion.matches;
-    let hovering = false;
-    let focused = false;
-    let visible = false;
-    let frame;
-    let previous;
-    let offset = viewport.scrollLeft;
-    function tick(time) {
-      const distance = viewport.scrollWidth - viewport.clientWidth;
-      if (distance <= 0) return;
-      if (previous !== undefined) {
-        offset = Math.min(offset + Math.min(time - previous, 50) * 0.035, distance);
-        viewport.scrollTo({ left: offset, behavior: 'instant' });
-      }
-      if (offset >= distance - 1) { paused = true; return; }
-      previous = time;
-      frame = window.requestAnimationFrame(tick);
-    }
-    function sync() {
-      window.cancelAnimationFrame(frame);
-      previous = undefined;
-      offset = viewport.scrollLeft;
-      if (!paused && !hovering && !focused && visible && !document.hidden) frame = window.requestAnimationFrame(tick);
-    }
-    viewport.addEventListener('pointerenter', event => { if (event.pointerType === 'mouse') { hovering = true; sync(); } });
-    viewport.addEventListener('pointerleave', () => { hovering = false; sync(); });
-    viewport.addEventListener('focusin', () => { focused = true; sync(); });
-    viewport.addEventListener('focusout', event => { focused = viewport.contains(event.relatedTarget); sync(); });
-    function pauseForInteraction() { paused = true; sync(); }
-    viewport.addEventListener('pointerdown', pauseForInteraction);
-    viewport.addEventListener('wheel', pauseForInteraction, { passive: true });
-    viewport.addEventListener('keydown', pauseForInteraction);
-    reducedMotion.addEventListener('change', () => { paused = reducedMotion.matches; sync(); });
-    document.addEventListener('visibilitychange', sync);
-    new window.IntersectionObserver(([entry]) => { visible = entry.isIntersecting; sync(); }).observe(gallery);
-    new window.ResizeObserver(sync).observe(viewport);
-    viewport.querySelectorAll('img').forEach(image => image.addEventListener('load', sync));
-  });
-})();

.github/pages/marquee.py

@@ -1,96 +0,0 @@
-"""Render homepage screenshot strips from the site's existing gallery HTML."""
-
-import re
-from html import escape
-from html.parser import HTMLParser
-from urllib.parse import urljoin
-
-
-class Gallery(HTMLParser):
-    """Only gallery figures are captures; navigation and decorative images are not."""
-
-    def __init__(self, base):
-        super().__init__()
-        self.base = base
-        self.stack = []
-        self.captures = []
-
-    def handle_starttag(self, tag, attrs):
-        attrs = dict(attrs)
-        if tag == "img" and any(name == "figure" for name, _ in self.stack):
-            context = [item for _, item in self.stack]
-            profile = next(
-                (
-                    item.get("data-profile") or item.get("data-viewport")
-                    for item in reversed(context)
-                    if item.get("data-profile") or item.get("data-viewport")
-                ),
-                None,
-            )
-            if profile not in (None, "desktop"):
-                return
-            source = attrs.get("src", "")
-            if not source or not attrs.get("alt"):
-                return
-            anchor = next((item["id"] for name, item in reversed(self.stack) if name in ("article", "figure") and item.get("id")), "")
-            server = any(item.get("data-platform") == "server" for item in context)
-            self.captures.append(
-                {
-                    "src": urljoin(self.base, source),
-                    "href": self.base + "#" + anchor if anchor else urljoin(self.base, source),
-                    "title": attrs["alt"],
-                    "width": attrs.get("width", ""),
-                    "height": attrs.get("height", ""),
-                    "product": "server" if server else "client",
-                },
-            )
-        if tag not in {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}:
-            self.stack.append((tag, attrs))
-
-    def handle_endtag(self, tag):
-        for index in range(len(self.stack) - 1, -1, -1):
-            if self.stack[index][0] == tag:
-                del self.stack[index:]
-                break
-
-
-def render(output, baseurl="", featured=()):
-    homepage = output / "index.html"
-    text = homepage.read_text()
-    slots = re.findall(r"<!-- ARCHIVEBOX:(MARQUEE(?:-CLIENT|-SERVER)?) -->", text)
-    if not slots:
-        return False
-    base = "/" + baseurl.strip("/") + "/" if baseurl.strip("/") else "/"
-    gallery = Gallery(base + "screenshots/")
-    gallery.feed((output / "screenshots/index.html").read_text())
-    for slot in slots:
-        product = slot.removeprefix("MARQUEE-").lower() if "-" in slot else None
-        captures = [capture for capture in gallery.captures if product is None or capture["product"] == product]
-        priority = {title: index for index, title in enumerate(featured)}
-        captures.sort(key=lambda capture: priority.get(capture["title"], len(priority)))
-        cards = []
-        for capture in captures:
-            dimensions = "".join(f' {key}="{escape(capture[key])}"' for key in ("width", "height") if capture[key])
-            cards.append(
-                f'<a class="abx-marquee-card" href="{escape(capture["href"])}"><img src="{escape(capture["src"])}" alt="{escape(capture["title"])}"{dimensions} loading="lazy" decoding="async"></a>',
-            )
-        viewport_id = slot.lower()
-        content = (
-            f'<section class="abx-marquee" aria-label="App screenshots"><div class="abx-marquee-viewport" id="{viewport_id}" tabindex="0" aria-label="Screenshots. Scroll sideways to explore."><div class="abx-marquee-track">{"".join(cards)}</div></div></section>'
-            if cards
-            else ""
-        )
-        start, end = f"<!-- ARCHIVEBOX:{slot} -->", f"<!-- /ARCHIVEBOX:{slot} -->"
-        replacement = start + "\n" + content + "\n" + end
-        if end in text:
-            text = re.sub(
-                re.escape(start) + r".*?" + re.escape(end),
-                lambda _, replacement=replacement: replacement,
-                text,
-                count=1,
-                flags=re.DOTALL,
-            )
-        else:
-            text = text.replace(start, replacement, 1)
-    homepage.write_text(text)
-    return True

.github/pages/nav.html

@@ -1,4 +0,0 @@
-<!-- Repository-specific navigation and primary CTA. -->
-<a href="__SITE_BASE__#use-cases">Use cases</a><a href="__SITE_BASE__#configuration">Configuration</a><a href="__SITE_BASE__#layout-security">Layout &amp; security</a><a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
-    <a href="https://github.com/ArchiveBox/ArchiveBox">GitHub ↗</a>
-    <a class="abx-cta" href="__SITE_BASE__#install">Get started</a>

.github/pages/site.json

@@ -1,16 +0,0 @@
-{
-  "name": "Web Archive",
-  "url": "https://archivebox.io/",
-  "pages": [
-    "index.html",
-    "screenshots/index.html"
-  ],
-  "marquee_featured": [
-    "Snapshots grid — desktop",
-    "Snapshot View (header collapsed) — desktop",
-    "AI agent — desktop",
-    "Archive results — desktop",
-    "Add URLs — desktop",
-    "Snapshots table — desktop"
-  ]
-}

.github/pages/site.py

@@ -1,215 +0,0 @@
-"""Assemble and verify this repository's static Pages site (standard library only)."""
-
-import argparse
-import json
-import re
-import shutil
-import subprocess
-from datetime import UTC, datetime
-from html import escape, unescape
-from html.parser import HTMLParser
-from pathlib import Path
-from urllib.parse import unquote, urlsplit
-
-from marquee import render as render_marquee
-
-HERE = Path(__file__).resolve().parent
-ROOT = HERE.parents[1]
-
-
-def replace_block(text, name, content):
-    start, end = f"<!-- ARCHIVEBOX:{name} -->", f"<!-- /ARCHIVEBOX:{name} -->"
-    pattern = re.escape(start) + r".*?" + re.escape(end)
-    if re.search(pattern, text, re.DOTALL):
-        return re.sub(
-            pattern,
-            lambda _: f"{start}\n{content}\n{end}",
-            text,
-            count=1,
-            flags=re.DOTALL,
-        )
-    if text.count(start) != 1:
-        raise ValueError(f"Expected one {name} placeholder")
-    return text.replace(start, f"{start}\n{content}\n{end}", 1)
-
-
-def render_metadata(text):
-    """Keep product copy in native templates; render one consistent SEO block."""
-    page = Page()
-    page.feed(text)
-    title = unescape(re.search(r"<title>(.*?)</title>", text, re.DOTALL)[1])
-    meta = page.metadata
-    description = meta.get("description", meta.get("og:description", ""))
-    image = meta.get("og:image", meta.get("twitter:image", ""))
-    if not description or not image or not page.canonical:
-        raise ValueError("Each page needs a description, social image and canonical")
-    values = {
-        "description": description,
-        "robots": "index,follow,max-image-preview:large",
-        "theme-color": "#9b2854",
-        "og:type": "website",
-        "og:site_name": meta.get("og:site_name", "ArchiveBox"),
-        "og:locale": meta.get("og:locale", "en_US"),
-        "og:url": page.canonical,
-        "og:title": title,
-        "og:description": description,
-        "og:image": image,
-        "og:image:type": meta.get("og:image:type", "image/png"),
-        "og:image:width": meta.get("og:image:width", "1200"),
-        "og:image:height": meta.get("og:image:height", "630"),
-        "og:image:alt": meta.get("og:image:alt", title),
-        "twitter:card": "summary_large_image",
-        "twitter:site": "@ArchiveBoxApp",
-        "twitter:title": title,
-        "twitter:description": description,
-        "twitter:image": image,
-        "twitter:image:alt": meta.get("og:image:alt", title),
-    }
-
-    def strip_managed_tag(match):
-        tag = Page()
-        tag.feed(match[0])
-        if tag.canonicals or any(key in values for key in tag.metadata):
-            return ""
-        # System fonts are shared across sites, without a network font dependency.
-        if "fonts.googleapis.com" in match[0] or "fonts.gstatic.com" in match[0]:
-            return ""
-        return match[0]
-
-    text = re.sub(r"<(?:meta|link)\b[^>]*>", strip_managed_tag, text)
-    text = re.sub(r"<title>.*?</title>", "", text, flags=re.DOTALL)
-    tags = [
-        f"<title>{escape(title)}</title>",
-        f'<link rel="canonical" href="{escape(page.canonical, quote=True)}">',
-    ]
-    for key, value in values.items():
-        attr = "property" if key.startswith("og:") else "name"
-        tags.append(f'<meta {attr}="{key}" content="{escape(value, quote=True)}">')
-    if "<!-- ARCHIVEBOX:METADATA -->" not in text:
-        text = text.replace("</head>", "<!-- ARCHIVEBOX:METADATA -->\n</head>", 1)
-    return replace_block(text, "METADATA", "\n".join(tags))
-
-
-def render(output, source=None, baseurl=""):
-    config = json.loads((HERE / "site.json").read_text())
-    output = output.resolve()
-    if source:
-        source = source.resolve()
-        if output == source or output in source.parents or source in output.parents:
-            raise ValueError("Build output must be separate from the source directory")
-        if output == ROOT or output in ROOT.parents:
-            raise ValueError("Build output must not replace a repository")
-        if output.exists():
-            shutil.rmtree(output)
-        shutil.copytree(source, output)
-    revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
-    assets = output / "site-base"
-    assets.mkdir(parents=True, exist_ok=True)
-    has_marquee = render_marquee(output, baseurl, config.get("marquee_featured", []))
-    common_assets = ["chrome.css", "chrome.js"] + (["marquee.css", "marquee.js"] if has_marquee else [])
-    for name in common_assets:
-        shutil.copyfile(HERE / "base" / name, assets / name)
-    for name in config["pages"]:
-        page = output / name
-        text = render_metadata(page.read_text())
-        base = "../" * (len(Path(name).parts) - 1) or "./"
-        header = (HERE / "base/header.html").read_text().replace("__SITE_NAME__", escape(config["name"]))
-        header = header.replace("__SITE_NAV__", (HERE / "nav.html").read_text().strip())
-        text = replace_block(text, "HEADER", header)
-        for kind in ["FOOTER-START", "FOOTER-END"]:
-            text = replace_block(text, kind, (HERE / "base" / f"{kind.lower()}.html").read_text().strip())
-        text = text.replace("__SITE_BASE__", base).replace("__SITE_REVISION__", revision)
-        resources = f'<link rel="stylesheet" href="{base}site-base/chrome.css?v={revision}">\n<script src="{base}site-base/chrome.js?v={revision}" defer></script>'
-        if has_marquee and name == "index.html":
-            resources += f'\n<link rel="stylesheet" href="{base}site-base/marquee.css?v={revision}">\n<script src="{base}site-base/marquee.js?v={revision}" defer></script>'
-        if "<!-- ARCHIVEBOX:ASSETS -->" not in text:
-            text = text.replace("</head>", "<!-- ARCHIVEBOX:ASSETS -->\n</head>", 1)
-        text = replace_block(text, "ASSETS", resources)
-        page.write_text(text)
-    # Site build provenance is independent of all original screenshot manifests.
-    provenance_path = output / "build.json"
-    provenance = json.loads(provenance_path.read_text()) if provenance_path.exists() else {}
-    provenance.update(
-        revision=revision,
-        generatedAt=datetime.now(UTC).isoformat(),
-        site=config["url"],
-    )
-    provenance_path.write_text(json.dumps(provenance, indent=2) + "\n")
-    (output / ".nojekyll").touch()
-    verify(output, baseurl)
-    print(f"Assembled {len(config['pages'])} pages at {output} from {revision}")
-
-
-class Page(HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self.ids = set()
-        self.metadata = {}
-        self.canonical = ""
-        self.resources = []
-        self.headers = self.footers = self.canonicals = 0
-
-    def handle_starttag(self, tag, attrs):
-        attrs = dict(attrs)
-        if tag == "meta":
-            for key in ["name", "property"]:
-                if attrs.get(key):
-                    self.metadata[attrs[key]] = attrs.get("content", "")
-        if tag == "link" and attrs.get("rel") == "canonical":
-            self.canonical = attrs["href"]
-        if attrs.get("id"):
-            self.ids.add(attrs["id"])
-        classes = attrs.get("class", "").split()
-        self.headers += "abx-header" in classes
-        self.footers += "abx-footer" in classes
-        self.canonicals += tag == "link" and attrs.get("rel") == "canonical"
-        if tag in ["img", "script"] and attrs.get("src"):
-            self.resources.append(attrs["src"])
-        if tag == "link" and attrs.get("rel") in [
-            "stylesheet",
-            "icon",
-            "apple-touch-icon",
-        ]:
-            self.resources.append(attrs["href"])
-
-
-def verify(output, baseurl=""):
-    config = json.loads((HERE / "site.json").read_text())
-    output = output.resolve()
-    prefix = "/" + baseurl.strip("/") if baseurl.strip("/") else ""
-    for name in config["pages"]:
-        path = output / name
-        text = path.read_text()
-        page = Page()
-        page.feed(text)
-        if (page.headers, page.footers, page.canonicals) != (1, 1, 1):
-            raise ValueError(f"{name}: expected exactly one shared header, footer and canonical")
-        if "__SITE_" in text:
-            raise ValueError(f"{name}: unresolved site placeholder")
-        for resource in page.resources:
-            url = urlsplit(resource)
-            if url.scheme or url.netloc or not url.path:
-                continue
-            resource_path = unquote(url.path)
-            if prefix and resource_path.startswith(prefix + "/"):
-                resource_path = resource_path[len(prefix) :]
-            local = (output / resource_path.lstrip("/") if resource_path.startswith("/") else path.parent / resource_path).resolve()
-            if not local.is_relative_to(output) or not local.is_file():
-                raise ValueError(f"{name}: missing local resource {resource}")
-        for asset in ["chrome.css", "chrome.js"]:
-            if (output / "site-base" / asset).read_bytes() != (HERE / "base" / asset).read_bytes():
-                raise ValueError(f"{name}: stale common asset {asset}")
-    print(f"Verified shared layout and local resources for {len(config['pages'])} pages")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("command", choices=["render", "verify"])
-    parser.add_argument("output", type=Path)
-    parser.add_argument("--source", type=Path)
-    parser.add_argument("--baseurl", default="")
-    args = parser.parse_args()
-    if args.command == "render":
-        render(args.output, args.source, args.baseurl)
-    else:
-        verify(args.output, args.baseurl)

.github/pages/verify.py

@@ -1,255 +0,0 @@
-"""Check the built site in real Chromium at desktop and phone widths."""
-
-import argparse
-import json
-import re
-from functools import partial
-from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
-from html.parser import HTMLParser
-from pathlib import Path
-from threading import Thread
-from urllib.parse import unquote, urljoin, urlsplit
-
-from playwright.sync_api import expect, sync_playwright
-
-HERE = Path(__file__).resolve().parent
-
-
-class Handler(SimpleHTTPRequestHandler):
-    def handle(self):
-        # Browser navigation can cancel an in-flight lazy image response.
-        try:
-            super().handle()
-        except (BrokenPipeError, ConnectionResetError):
-            pass
-
-    def log_message(self, *args):
-        pass
-
-
-class ScreenshotReferences(HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self.references = []
-        self.ids = set()
-
-    def handle_starttag(self, tag, attributes):
-        attrs = dict(attributes)
-        if attrs.get("id"):
-            self.ids.add(attrs["id"])
-        for attribute in ("src", "href"):
-            if attrs.get(attribute):
-                self.references.append(attrs[attribute])
-
-
-def verify_screenshot_references(output, canonical):
-    """Check gallery embeds on every static page, including optional translations."""
-    pages = {}
-    for path in output.rglob("*.html"):
-        page = ScreenshotReferences()
-        page.feed(path.read_text())
-        pages[path.resolve()] = page
-    for path, page in pages.items():
-        page_url = urljoin(canonical, path.relative_to(output.resolve()).as_posix())
-        for reference in page.references:
-            url = urlsplit(urljoin(page_url, reference))
-            if url.netloc != urlsplit(canonical).netloc or "/screenshots/" not in url.path:
-                continue
-            target = output / unquote(url.path).lstrip("/")
-            if target.is_dir():
-                target /= "index.html"
-            target = target.resolve()
-            assert target.is_relative_to(output.resolve()) and target.is_file(), f"{path}: missing screenshot target {reference}"
-            assert not re.match(r"^\d+-", target.name), f"{path}: screenshot filename depends on capture order: {reference}"
-            if url.fragment and target in pages:
-                assert unquote(url.fragment) in pages[target].ids, f"{path}: missing screenshot anchor {reference}"
-
-
-def verify(output, evidence):
-    config = json.loads((HERE / "site.json").read_text())
-    verify_screenshot_references(output, config["url"])
-    evidence.mkdir(parents=True, exist_ok=True)
-    server = ThreadingHTTPServer(("127.0.0.1", 0), partial(Handler, directory=str(output.resolve())))
-    thread = Thread(target=server.serve_forever, daemon=True)
-    thread.start()
-    origin = f"http://127.0.0.1:{server.server_port}"
-    try:
-        with sync_playwright() as playwright:
-            browser = playwright.chromium.launch(ignore_default_args=["--hide-scrollbars"])
-            for name in config["pages"]:
-                route = name.removesuffix("index.html")
-                page = browser.new_page(reduced_motion="reduce")
-                errors = []
-                missing = []
-                page.on("pageerror", lambda error, errors=errors: errors.append(str(error)))
-                page.on(
-                    "response",
-                    lambda response, missing=missing: (
-                        missing.append(response.url) if response.url.startswith(origin) and response.status >= 400 else None
-                    ),
-                )
-                for width in [1716, 2560, 390]:
-                    page.set_viewport_size({"width": width, "height": 900})
-                    page.goto(f"{origin}/{route}", wait_until="domcontentloaded")
-                    header = page.locator(".abx-header")
-                    expect(header).to_have_count(1)
-                    expect(header).to_be_visible()
-                    expect(header).to_be_in_viewport()
-                    brand = header.locator(".abx-brand")
-                    expect(brand).to_be_visible()
-                    expect(brand).to_contain_text("ArchiveBox")
-                    expect(brand.locator(".abx-logo")).to_be_visible()
-                    expect(brand.locator(".abx-logo")).to_be_in_viewport()
-                    expect(header).to_have_css("display", "flex")
-                    expect(page.locator(".abx-footer")).to_have_count(1)
-                    expect(page.locator(".abx-footer-column")).to_have_count(3)
-                    expect(header.locator(".abx-cta")).to_be_visible()
-                    menu = header.locator(".abx-apps")
-                    menu.locator("summary").focus()
-                    page.keyboard.press("Enter")
-                    expect(menu).to_have_attribute("open", "")
-                    expect(menu.locator("a").first).to_be_visible()
-                    page.keyboard.press("Escape")
-                    expect(menu).not_to_have_attribute("open", "")
-                    menu.locator("summary").click()
-                    box = header.bounding_box()
-                    assert box
-                    page.mouse.click(box["x"] + 2, box["y"] + 5)
-                    expect(menu).not_to_have_attribute("open", "")
-                    assert page.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth"), (
-                        f"{name}: overflow at {width}"
-                    )
-                    if not route and "screenshots/index.html" in config["pages"]:
-                        strips = page.locator(".abx-marquee")
-                        assert strips.count() > 0, "Homepage must use its screenshot gallery"
-                        for strip in strips.all():
-                            strip.scroll_into_view_if_needed()
-                            bounds = strip.bounding_box()
-                            usable_width = page.evaluate("document.documentElement.clientWidth")
-                            assert bounds and abs(bounds["x"]) < 1
-                            assert abs(bounds["width"] - usable_width) < 1
-                            assert not strip.inner_text().strip(), "Screenshot strips should have no visible captions or links"
-                            cards = strip.locator(".abx-marquee-card")
-                            assert cards.count() > 0, "Screenshot strip is empty"
-                            expect(strip.locator("button")).to_have_count(0)
-                            expect(strip.locator("a:not(.abx-marquee-card)")).to_have_count(0)
-                            viewport = strip.locator(".abx-marquee-viewport")
-                            overflowing = viewport.evaluate(
-                                "node => node.scrollWidth > node.clientWidth",
-                            )
-                            if overflowing:
-                                page.mouse.move(0, 0)
-                                page.emulate_media(reduced_motion="no-preference")
-                                initial = viewport.evaluate("node => node.scrollLeft")
-                                page.wait_for_function(
-                                    "([node, initial]) => node.scrollLeft > initial",
-                                    arg=[viewport.element_handle(), initial],
-                                )
-                                viewport.hover()
-                                stopped = viewport.evaluate("node => node.scrollLeft")
-                                viewport.evaluate(
-                                    "node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
-                                )
-                                assert viewport.evaluate("node => node.scrollLeft") == stopped
-                                page.emulate_media(reduced_motion="reduce")
-                                page.mouse.move(0, 0)
-                                stopped = viewport.evaluate("node => node.scrollLeft")
-                                viewport.evaluate(
-                                    "node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
-                                )
-                                assert viewport.evaluate("node => node.scrollLeft") == stopped
-                                viewport.focus()
-                                page.keyboard.press("ArrowRight")
-                                assert viewport.evaluate("node => node.scrollWidth > node.clientWidth")
-                            else:
-                                centered = viewport.evaluate(
-                                    """node => {
-                                        const view = node.getBoundingClientRect();
-                                        const track = node.querySelector('.abx-marquee-track').getBoundingClientRect();
-                                        return Math.abs((track.left - view.left) - (view.right - track.right)) < 2;
-                                    }""",
-                                )
-                                assert centered, "Short screenshot strips must be centered"
-                            for card in cards.all():
-                                href = card.get_attribute("href")
-                                assert href and "/screenshots/" in href
-                                response = page.request.get(urljoin(page.url, href))
-                                assert response.ok, f"Broken screenshot link: {href}"
-                                anchor = urlsplit(href).fragment
-                                if anchor:
-                                    assert f'id="{anchor}"' in response.text(), f"Missing screenshot anchor: {href}"
-                    # Decode every local content image using the real browser loader,
-                    # including images in offscreen carousel tracks and lazy galleries.
-                    images = page.locator("main img").evaluate_all("nodes => [...new Set(nodes.map(image => image.src))]")
-                    local_images = [url for url in images if url.startswith(origin + "/")]
-                    page.evaluate(
-                        """async urls => {
-                        for (const url of urls) {
-                            const image = new Image();
-                            image.src = url;
-                            await image.decode();
-                            if (!image.naturalWidth) throw new Error(`Broken image: ${url}`);
-                        }
-                    }""",
-                        local_images,
-                    )
-                    page.evaluate("scrollTo({top: 0, behavior: 'instant'})")
-                    page.evaluate(
-                        "() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
-                    )
-                    page.screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}{width}.png"))
-                    page.locator(".abx-footer").screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}footer-{width}.png"))
-                    for link in header.locator(".abx-nav > a").all():
-                        url = urlsplit(link.get_attribute("href") or "")
-                        if url.scheme or url.netloc:
-                            continue
-                        response = page.request.get(urljoin(page.url, url.geturl()))
-                        assert response.ok, f"Broken navigation: {url.geturl()}"
-                        if url.fragment:
-                            body = response.text()
-                            assert f'id="{url.fragment}"' in body or f"id='{url.fragment}'" in body, (
-                                f"Missing navigation anchor: {url.geturl()}"
-                            )
-                    if route == "" and page.locator("#resources").count():
-                        page.goto(f"{origin}/#resources")
-                        expect(page.locator("#resources")).to_be_visible()
-                assert not errors, errors
-                assert not missing, missing
-                page.close()
-                context = browser.new_context(java_script_enabled=False, viewport={"width": 390, "height": 900})
-                plain = context.new_page()
-                plain.goto(f"{origin}/{route}", wait_until="domcontentloaded")
-                expect(plain.locator(".abx-header")).to_be_visible()
-                expect(plain.locator(".abx-header .abx-brand")).to_be_in_viewport()
-                expect(plain.locator(".abx-header .abx-logo")).to_be_visible()
-                plain.locator(".abx-apps summary").click()
-                expect(plain.locator(".abx-app-links a").first).to_be_visible()
-                expect(plain.locator(".abx-footer-column a").first).to_be_visible()
-                if not route and "screenshots/index.html" in config["pages"]:
-                    assert plain.locator(".abx-marquee-card").count() > 0
-                    for strip in plain.locator(".abx-marquee").all():
-                        bounds = strip.bounding_box()
-                        usable_width = plain.evaluate("document.documentElement.clientWidth")
-                        assert bounds and abs(bounds["x"]) < 1
-                        assert abs(bounds["width"] - usable_width) < 1
-                    assert plain.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth")
-                    expect(plain.locator(".abx-marquee button")).to_have_count(0)
-                    expect(plain.locator(".abx-marquee-card").first).to_be_visible()
-                context.close()
-                print(
-                    f"PASS {name}: desktop/mobile, keyboard, no-JS, images and resources",
-                    flush=True,
-                )
-            browser.close()
-    finally:
-        server.shutdown()
-        server.server_close()
-        thread.join()
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("output", type=Path)
-    parser.add_argument("--evidence", type=Path, default=Path("test-results/site"))
-    args = parser.parse_args()
-    verify(args.output, args.evidence)

.github/scripts/discover_test_matrix.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python3
-"""Discover every test once, batching measured short files."""
-
-import json
-import os
-from pathlib import Path
-
-
-def main() -> None:
-    root = Path(__file__).resolve().parents[2]
-    archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
-
-    if not archivebox_tests:
-        raise SystemExit("No ArchiveBox tests discovered")
-
-    # These are observed per-file pytest seconds, including fixture/startup
-    # time. Shared job setup is paid once per batch. Unknown/long files, optional
-    # dependencies and hosted-only requirements keep independent jobs.
-    durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
-    hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
-    requested = os.environ.get("TEST_FILE", "")
-    batches: list[list[str]] = []
-    batch: list[str] = []
-    batch_seconds = 0
-    for path in archivebox_tests:
-        test_path = path.relative_to(root).as_posix()
-        duration = durations.get(test_path, 60)
-        if duration >= 60 or path in hosted or path.stem == "test_auth_ldap" or test_path == requested:
-            batches.append([test_path])
-            continue
-        if batch and (len(batch) == 8 or batch_seconds + duration > 180):
-            batches.append(batch)
-            batch = []
-            batch_seconds = 0
-        batch.append(test_path)
-        batch_seconds += duration
-    if batch:
-        batches.append(batch)
-
-    matrix: list[dict[str, object]] = []
-    for paths in batches:
-        matrix.append(
-            {
-                "name": f"main/{Path(paths[0]).stem}" if len(paths) == 1 else f"batch/{Path(paths[0]).stem}",
-                "paths": paths,
-                "paths_arg": " ".join(paths),
-                "extra": "ldap" if paths == ["archivebox/tests/test_auth_ldap.py"] else "",
-                "count": len(paths),
-                "ugnas": False,
-            },
-        )
-
-    # Ordinary Linux tests can use either pool. Only genuine hosted-only
-    # requirements need a file header; filenames never enter runner policy.
-    # Keep a bounded NAS share so hosted capacity continues working in parallel.
-    # This is queued work, not a worker count: several waves keep the capped
-    # runners occupied while hosted jobs run in parallel.
-    nas_job_limit = int(os.environ.get("UGNAS_CI_MAX_JOBS", "3"))
-    # Keep long jobs early in the hosted queue. Giving the longest jobs to the
-    # CPU-capped NAS instead created a tail after hosted work had finished; use
-    # its extra slots for a bounded share of shorter work without filename rules.
-    matrix.sort(key=lambda item: sum(durations.get(path, 60) for path in item["paths"]), reverse=True)
-    eligible = [item for item in reversed(matrix) if not any(root / path in hosted for path in item["paths"])]
-    for item in eligible[:nas_job_limit]:
-        item["ugnas"] = True
-
-    discovered_paths = [path for entry in matrix for path in entry["paths"]]
-    if len(discovered_paths) != len(set(discovered_paths)):
-        raise SystemExit("Tests were not discovered exactly once")
-    if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
-        raise SystemExit("Discovered test shard coverage does not match test files")
-
-    print(
-        f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs; "
-        f"{sum(item['ugnas'] for item in matrix)} selected for ugNAS, subject to the availability check",
-    )
-    print(json.dumps(matrix, separators=(",", ":")))
-
-
-if __name__ == "__main__":
-    main()

.github/test-durations.json

@@ -1,174 +0,0 @@
-{
-  "source_runs": [
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37190995942"
-  ],
-  "measurement": "Per-file pytest wall seconds from JUnit suites, including fixture setup, plus 9 seconds for process startup outside JUnit (observed median 8.1 seconds across 69 successful unbatched jobs). Failed files retain the larger prior successful estimate. Unknown files stay independent. Queue and shared job setup are excluded.",
-  "seconds": {
-    "archivebox/tests/test_api_discovery_performance.py": 28,
-    "archivebox/tests/test_api_v1_auth_browser_session.py": 17,
-    "archivebox/tests/test_api_v1_auth_check_api_token.py": 15,
-    "archivebox/tests/test_api_v1_auth_get_api_token.py": 16,
-    "archivebox/tests/test_api_v1_cli_add.py": 84,
-    "archivebox/tests/test_api_v1_cli_remove.py": 33,
-    "archivebox/tests/test_api_v1_cli_schedule.py": 29,
-    "archivebox/tests/test_api_v1_cli_search.py": 15,
-    "archivebox/tests/test_api_v1_cli_update.py": 175,
-    "archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 81,
-    "archivebox/tests/test_api_v1_core_any_id.py": 13,
-    "archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 24,
-    "archivebox/tests/test_api_v1_core_archiveresults.py": 19,
-    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 106,
-    "archivebox/tests/test_api_v1_core_snapshots.py": 17,
-    "archivebox/tests/test_api_v1_core_snapshots_rss.py": 16,
-    "archivebox/tests/test_api_v1_core_tag_tag_id.py": 15,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 16,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 16,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 16,
-    "archivebox/tests/test_api_v1_core_tags.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 16,
-    "archivebox/tests/test_api_v1_core_tags_autocomplete.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_create.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_search.py": 17,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 131,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 15,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 15,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 16,
-    "archivebox/tests/test_api_v1_crawls_crawls.py": 16,
-    "archivebox/tests/test_api_v1_machine_binaries.py": 16,
-    "archivebox/tests/test_api_v1_machine_binary_binary_id.py": 16,
-    "archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 16,
-    "archivebox/tests/test_api_v1_machine_machine_current.py": 15,
-    "archivebox/tests/test_api_v1_machine_machine_machine_id.py": 16,
-    "archivebox/tests/test_api_v1_machine_machines.py": 15,
-    "archivebox/tests/test_api_v1_personas_personas.py": 15,
-    "archivebox/tests/test_api_v1_personas_sync.py": 16,
-    "archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 27,
-    "archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 19,
-    "archivebox/tests/test_archive_result_service.py": 49,
-    "archivebox/tests/test_auth_ldap.py": 32,
-    "archivebox/tests/test_binary_service.py": 109,
-    "archivebox/tests/test_cli_add_1.py": 107,
-    "archivebox/tests/test_cli_add_2.py": 86,
-    "archivebox/tests/test_cli_add_3.py": 108,
-    "archivebox/tests/test_cli_add_4.py": 105,
-    "archivebox/tests/test_cli_archiveresult.py": 86,
-    "archivebox/tests/test_cli_background_screenshot.py": 49,
-    "archivebox/tests/test_cli_binary.py": 12,
-    "archivebox/tests/test_cli_config.py": 75,
-    "archivebox/tests/test_cli_crawl.py": 98,
-    "archivebox/tests/test_cli_extract.py": 63,
-    "archivebox/tests/test_cli_extract_input.py": 124,
-    "archivebox/tests/test_cli_help.py": 16,
-    "archivebox/tests/test_cli_init.py": 72,
-    "archivebox/tests/test_cli_install.py": 38,
-    "archivebox/tests/test_cli_list.py": 141,
-    "archivebox/tests/test_cli_machine.py": 19,
-    "archivebox/tests/test_cli_manage.py": 32,
-    "archivebox/tests/test_cli_mcp.py": 30,
-    "archivebox/tests/test_cli_oneshot.py": 15,
-    "archivebox/tests/test_cli_persona.py": 147,
-    "archivebox/tests/test_cli_piping.py": 60,
-    "archivebox/tests/test_cli_pluginmap.py": 12,
-    "archivebox/tests/test_cli_process.py": 12,
-    "archivebox/tests/test_cli_remove.py": 102,
-    "archivebox/tests/test_cli_run_1.py": 117,
-    "archivebox/tests/test_cli_run_2.py": 112,
-    "archivebox/tests/test_cli_run_3.py": 80,
-    "archivebox/tests/test_cli_run_4.py": 111,
-    "archivebox/tests/test_cli_schedule.py": 147,
-    "archivebox/tests/test_cli_search.py": 25,
-    "archivebox/tests/test_cli_server.py": 143,
-    "archivebox/tests/test_cli_shell.py": 17,
-    "archivebox/tests/test_cli_snapshot.py": 109,
-    "archivebox/tests/test_cli_status.py": 46,
-    "archivebox/tests/test_cli_tag.py": 12,
-    "archivebox/tests/test_cli_update.py": 114,
-    "archivebox/tests/test_cli_update_reindex_snapshots.py": 100,
-    "archivebox/tests/test_cli_version.py": 119,
-    "archivebox/tests/test_config_DELETE_AFTER.py": 187,
-    "archivebox/tests/test_config_MAX_limits.py": 25,
-    "archivebox/tests/test_config_ONLY_NEW.py": 15,
-    "archivebox/tests/test_config_SAVE_TITLE.py": 59,
-    "archivebox/tests/test_config_URL_filters.py": 15,
-    "archivebox/tests/test_core_config.py": 11,
-    "archivebox/tests/test_core_forms.py": 14,
-    "archivebox/tests/test_crawl_history.py": 162,
-    "archivebox/tests/test_crawl_pause.py": 11,
-    "archivebox/tests/test_crawl_runner.py": 22,
-    "archivebox/tests/test_crawl_runner_2.py": 27,
-    "archivebox/tests/test_crawl_service.py": 19,
-    "archivebox/tests/test_frozen_crawl_config.py": 26,
-    "archivebox/tests/test_hook_result_lifecycle.py": 83,
-    "archivebox/tests/test_hooks.py": 54,
-    "archivebox/tests/test_issue_1896.py": 107,
-    "archivebox/tests/test_machine_models.py": 26,
-    "archivebox/tests/test_machine_service.py": 87,
-    "archivebox/tests/test_migrations_04_to_09.py": 25,
-    "archivebox/tests/test_migrations_07_to_09.py": 107,
-    "archivebox/tests/test_migrations_08_to_09.py": 141,
-    "archivebox/tests/test_migrations_fresh.py": 55,
-    "archivebox/tests/test_misc_checks.py": 12,
-    "archivebox/tests/test_misc_monkey_patches.py": 12,
-    "archivebox/tests/test_opencode_agent_1.py": 92,
-    "archivebox/tests/test_opencode_agent_2.py": 60,
-    "archivebox/tests/test_opencode_agent_3.py": 154,
-    "archivebox/tests/test_opencode_browser_1.py": 268,
-    "archivebox/tests/test_opencode_browser_2.py": 77,
-    "archivebox/tests/test_opencode_browser_3.py": 74,
-    "archivebox/tests/test_orm_helpers.py": 14,
-    "archivebox/tests/test_permissions.py": 12,
-    "archivebox/tests/test_persona_casing.py": 96,
-    "archivebox/tests/test_persona_runtime.py": 30,
-    "archivebox/tests/test_postgres_backend.py": 76,
-    "archivebox/tests/test_process_runtime_paths.py": 20,
-    "archivebox/tests/test_process_service.py": 40,
-    "archivebox/tests/test_proof_previews.py": 24,
-    "archivebox/tests/test_recursive_crawl_1.py": 374,
-    "archivebox/tests/test_recursive_crawl_2.py": 55,
-    "archivebox/tests/test_recursive_crawl_3.py": 82,
-    "archivebox/tests/test_release_notes.py": 12,
-    "archivebox/tests/test_release_source_fingerprint.py": 13,
-    "archivebox/tests/test_release_workflow.py": 15,
-    "archivebox/tests/test_resource_admission.py": 13,
-    "archivebox/tests/test_runner_deletion.py": 135,
-    "archivebox/tests/test_runner_watch.py": 104,
-    "archivebox/tests/test_search.py": 151,
-    "archivebox/tests/test_serve_static.py": 19,
-    "archivebox/tests/test_server_automatic_url.py": 22,
-    "archivebox/tests/test_server_security_browser.py": 137,
-    "archivebox/tests/test_settings_signal_webhooks.py": 13,
-    "archivebox/tests/test_setup_script.py": 12,
-    "archivebox/tests/test_shutdown_util.py": 13,
-    "archivebox/tests/test_snapshot_filesystem_migration.py": 16,
-    "archivebox/tests/test_snapshot_output_stacks.py": 76,
-    "archivebox/tests/test_snapshot_ref_lookup.py": 15,
-    "archivebox/tests/test_snapshot_service.py": 48,
-    "archivebox/tests/test_snapshot_url_length.py": 16,
-    "archivebox/tests/test_tag_service.py": 15,
-    "archivebox/tests/test_takeover_util_1.py": 100,
-    "archivebox/tests/test_takeover_util_2.py": 86,
-    "archivebox/tests/test_takeover_util_3.py": 98,
-    "archivebox/tests/test_takeover_util_4.py": 96,
-    "archivebox/tests/test_test_harness.py": 12,
-    "archivebox/tests/test_ui_add_view.py": 26,
-    "archivebox/tests/test_ui_add_view_runtime.py": 121,
-    "archivebox/tests/test_ui_admin_apitoken.py": 16,
-    "archivebox/tests/test_ui_admin_archiveresult.py": 18,
-    "archivebox/tests/test_ui_admin_config_widget.py": 13,
-    "archivebox/tests/test_ui_admin_crawl.py": 25,
-    "archivebox/tests/test_ui_admin_links.py": 23,
-    "archivebox/tests/test_ui_admin_machine.py": 18,
-    "archivebox/tests/test_ui_admin_persona.py": 19,
-    "archivebox/tests/test_ui_admin_snapshot.py": 116,
-    "archivebox/tests/test_ui_admin_tag.py": 16,
-    "archivebox/tests/test_ui_admin_user.py": 16,
-    "archivebox/tests/test_ui_config_views.py": 20,
-    "archivebox/tests/test_ui_live_progress.py": 140,
-    "archivebox/tests/test_ui_public_snapshot.py": 95,
-    "archivebox/tests/test_ui_savepagenow.py": 31,
-    "archivebox/tests/test_urls.py": 132,
-    "archivebox/tests/test_util.py": 13,
-    "archivebox/tests/test_version_metadata.py": 13
-  }
-}

.github/workflows/ci.yml

@@ -1,103 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-    branches: [dev, main]
-  push:
-    branches: [dev, main]
-  workflow_dispatch:
-
-permissions:
-  actions: read
-  contents: read
-  packages: write
-  security-events: write
-
-concurrency:
-  group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }}
-  cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
-
-jobs:
-  prepare:
-    permissions:
-      contents: write
-    uses: ./.github/workflows/prepare-release.yml
-    secrets: inherit
-
-  lint:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Linters
-    uses: ./.github/workflows/lint.yml
-    secrets: inherit
-
-  install-cli-platform:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Install and CLI platform compatibility
-    uses: ./.github/workflows/test.yml
-    secrets: inherit
-
-  tests:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Discovered test matrix
-    uses: ./.github/workflows/test-parallel.yml
-    secrets: inherit
-
-  codeql:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: CodeQL
-    uses: ./.github/workflows/codeql.yml
-    secrets: inherit
-
-  python-artifacts:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Tested Python artifacts
-    uses: ./.github/workflows/pip.yml
-    secrets: inherit
-
-  docker-digests:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Tested Docker images
-    uses: ./.github/workflows/docker.yml
-    with:
-      push_digests: ${{ github.event_name == 'push' && (github.ref_name == 'dev' || github.ref_name == 'main') }}
-    secrets: inherit
-
-  required:
-    name: All required CI lanes
-    if: ${{ always() && needs.prepare.outputs.run_jobs == 'true' }}
-    needs:
-      - prepare
-      - lint
-      - install-cli-platform
-      - tests
-      - codeql
-      - python-artifacts
-      - docker-digests
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Verify every required lane succeeded
-        env:
-          LINT_RESULT: ${{ needs.lint.result }}
-          INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
-          TESTS_RESULT: ${{ needs.tests.result }}
-          CODEQL_RESULT: ${{ needs.codeql.result }}
-          PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
-          DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
-        run: |
-          for result in \
-            "$LINT_RESULT" \
-            "$INSTALL_CLI_PLATFORM_RESULT" \
-            "$TESTS_RESULT" \
-            "$CODEQL_RESULT" \
-            "$PYTHON_ARTIFACTS_RESULT" \
-            "$DOCKER_DIGESTS_RESULT"
-          do
-            [[ "$result" == success ]]
-          done
-          echo 'All required CI lanes and tested release artifacts succeeded.'

.github/workflows/claude.yml

@@ -1,40 +0,0 @@
-name: Claude Code
-
-on:
-  issue_comment:
-    types: [created]
-  pull_request_review_comment:
-    types: [created]
-  issues:
-    types: [opened, assigned]
-  pull_request_review:
-    types: [submitted]
-
-jobs:
-  claude:
-    if: |
-      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
-      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
-      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: read
-      pull-requests: write
-      issues: write
-      id-token: write
-      actions: read
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Run Claude Code
-        id: claude
-        uses: anthropics/claude-code-action@b76a0776ae74036e77cd11018083743453d7ad35 # v1
-        with:
-          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
-          additional_permissions: |
-            actions: read
-          claude_args: '--allowed-tools Bash(gh pr:*)'

.github/workflows/codeql.yml

@@ -1,21 +1,42 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
 name: "CodeQL"
 
 on:
-  workflow_call:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
   schedule:
     - cron: '33 17 * * 6'
 
-concurrency:
-  group: codeql-${{ github.repository }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
   analyze:
     name: Analyze (${{ matrix.language }})
-    runs-on: ubuntu-24.04
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
     permissions:
+      # required for all workflows
       security-events: write
+
+      # required to fetch internal or private CodeQL packs
       packages: read
+
+      # only required for workflows in private repositories
       actions: read
       contents: read
 
@@ -25,18 +46,47 @@ jobs:
         include:
         - language: python
           build-mode: none
-        - language: javascript-typescript
-          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
     steps:
     - name: Checkout repository
-      uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      uses: actions/checkout@v4
 
+    # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/debian.yml

@@ -0,0 +1,64 @@
+name: Build Debian package
+
+on:
+  workflow_dispatch:
+  push:
+
+env:
+  DEB_BUILD_OPTIONS: nocheck
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Install packaging dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y \
+            python3 python3-dev python3-pip python3-venv python3-all \
+            dh-python debhelper devscripts dput software-properties-common \
+            python3-distutils python3-setuptools python3-wheel python3-stdeb
+
+      # - name: Build Debian/Apt sdist_dsc
+      #   run: |
+      #     ./bin/build_pip.sh
+
+      # - name: Check ArchiveBox version
+      #   run: |
+      #     # must create dir needed for snaps to run as non-root on github actions
+      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
+      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
+      #     archivebox --version
+      #     archivebox init --setup
+
+      # - name: Add some links to test
+      #   run: |
+      #     cd "${{ github.workspace }}/data"
+      #     archivebox add 'https://example.com'
+      #     archivebox status
+
+      # - name: Commit built package
+      #   run: |
+      #     cd deb_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Debian package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/debian-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: deb_dist
+
+      # - name: Push build to Launchpad PPA
+      #   run: |
+      #     debsign -k "$PGP_KEY_ID" "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
+      #     dput archivebox "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"

.github/workflows/deploy-publicsite.yml

@@ -1,87 +0,0 @@
-name: Publish website
-
-on:
-  push:
-    branches: [dev]
-  pull_request:
-  workflow_dispatch:
-  workflow_run:
-    workflows: ['Capture publicsite screenshots']
-    types: [completed]
-    branches: [dev]
-
-permissions:
-  contents: read
-  actions: read
-
-env:
-  # Repository-specific output; all publication logic below is a local copy.
-  SITE_OUTPUT: _site
-  DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-
-# One production queue covers restore, build and deploy. PRs have separate queues.
-concurrency:
-  group: pages-${{ github.event_name == 'pull_request' && github.ref || 'production' }}
-  cancel-in-progress: false
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    outputs:
-      revision: ${{ steps.source.outputs.revision }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event_name == 'pull_request' && github.ref || env.DEFAULT_BRANCH }}
-          persist-credentials: false
-      - name: Record website source revision
-        id: source
-        run: echo "revision=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
-
-      # BEGIN REPOSITORY-SPECIFIC BUILD
-      - uses: astral-sh/setup-uv@v7
-      - name: Restore latest complete screenshots
-        run: uv run --no-project python bin/restore_site_screenshots.py publicsite/screenshots
-        env:
-          GH_TOKEN: ${{ github.token }}
-      - name: Render current gallery layout without recapturing images
-        run: uv run --no-project python bin/generate_ui_screenshot_gallery.py refresh publicsite/screenshots/index.html
-      - name: Assemble website with the local base components
-        run: uv run --no-project python .github/pages/site.py render _site --source publicsite
-      # END REPOSITORY-SPECIFIC BUILD
-
-      - name: Verify website in Chromium
-        run: |
-          uv run --no-config --no-project --with playwright==1.63.0 playwright install --with-deps chromium
-          uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py "$SITE_OUTPUT" --evidence "$RUNNER_TEMP/site-evidence"
-      - name: Save website verification evidence
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: site-verification
-          path: ${{ runner.temp }}/site-evidence
-          if-no-files-found: warn
-          retention-days: 14
-
-      - name: Upload website
-        uses: actions/upload-pages-artifact@v3
-        with:
-          path: ${{ env.SITE_OUTPUT }}
-
-  deploy:
-    if: github.event_name != 'pull_request'
-    needs: build
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      pages: write
-      id-token: write
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    steps:
-      - uses: actions/configure-pages@v5
-      - uses: actions/deploy-pages@v4
-        id: deployment

.github/workflows/docker.yml

@@ -1,143 +1,37 @@
 name: Build Docker image
 
 on:
-  workflow_call:
-    inputs:
-      push_digests:
-        description: Push validated per-platform images by digest for the release workflow
-        required: false
-        default: false
-        type: boolean
-      full_tests:
-        description: Run storage, CLI, and size validation in addition to release smoke checks
-        required: false
-        default: true
-        type: boolean
   workflow_dispatch:
-    inputs:
-      source_ref:
-        description: Exact 40-character commit SHA to build
-        required: true
-        type: string
-      push_digests:
-        description: Push validated per-platform images by digest for the release workflow
-        required: false
-        default: true
-        type: boolean
-      full_tests:
-        description: Run storage, CLI, and size validation in addition to release smoke checks
-        required: false
-        default: true
-        type: boolean
+  push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'
+  # pull_request:
 
 env:
-  DOCKERHUB_IMAGE: archivebox/archivebox
-  GHCR_IMAGE: ghcr.io/archivebox/archivebox
-
-permissions:
-  actions: read
-  contents: read
-  packages: write
+  DOCKER_IMAGE: archivebox-ci
 
 jobs:
-  build:
-    name: build ${{ matrix.platform }}
-    runs-on: ${{ matrix.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: linux/amd64
-            runner: ubuntu-24.04
-            artifact_name: digest-linux-amd64
-            local_tag: archivebox-ci:amd64
-          - platform: linux/arm64
-            runner: ubuntu-24.04-arm
-            artifact_name: digest-linux-arm64
-            local_tag: archivebox-ci:arm64
+  buildx:
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.source_ref || github.sha }}
+          submodules: true
+          fetch-depth: 1
 
-      - name: Verify exact source commit
-        if: github.event_name == 'workflow_dispatch'
-        shell: bash
-        env:
-          SOURCE_REF: ${{ inputs.source_ref || github.sha }}
-        run: |
-          set -Eeuo pipefail
-          [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
-          [[ "$(git rev-parse HEAD)" == "$SOURCE_REF" ]]
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Resolve Docker through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python - <<'PY'
-          import tomllib
-          from pathlib import Path
-
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          matches = [package for package in packages if package["name"] == "abxpkg"]
-          if len(matches) != 1:
-              raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}")
-          wheel = matches[0]["wheels"][0]
-          print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))
-          PY
-          )"
-          test -n "$ABXPKG_SPEC"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_SPEC=$ABXPKG_SPEC" >> "$GITHUB_ENV"
-          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-
-          docker_env="$(
-            uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries"
-          )"
-          JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$docker_env" >> "$GITHUB_ENV"
-
-          docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
-          UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv"
-          {
-            echo "DOCKER_BINARY=$docker_binary"
-            echo "JQ_BINARY=$JQ_BINARY"
-            echo "UV_BINARY=$UV_BINARY"
-          } >> "$GITHUB_ENV"
-          for binary in "$docker_binary" "$JQ_BINARY" "$UV_BINARY"; do
-            test -L "$binary"
-            test -x "$binary"
-          done
-          "$docker_binary" --version
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+        uses: docker/setup-buildx-action@v3
         with:
+          version: latest
           install: true
+          platforms: linux/amd64,linux/arm64
 
       - name: Builder instance name
         run: echo ${{ steps.buildx.outputs.name }}
@@ -145,440 +39,68 @@ jobs:
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
-      - name: Verify published abx-dl image
-        id: abx_dl_image
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABX_DL_VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import tomllib
-          from pathlib import Path
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
 
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          matches = [package["version"] for package in packages if package["name"] == "abx-dl"]
-          if len(matches) != 1:
-              raise SystemExit(f"Expected one locked abx-dl package, found {len(matches)}")
-          print(matches[0])
-          PY
-          )"
-          test -n "$ABX_DL_VERSION"
-          ABX_DL_REFERENCE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
-          INSPECT_OUTPUT="$("$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_REFERENCE")"
-          printf '%s\n' "$INSPECT_OUTPUT"
-          ABX_DL_DIGEST="$(INSPECT_OUTPUT="$INSPECT_OUTPUT" "$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import os
-
-          digest = next(
-              (fields[1] for line in os.environ["INSPECT_OUTPUT"].splitlines() if (fields := line.split()) and fields[0] == "Digest:"),
-              "",
-          )
-          print(digest)
-          PY
-          )"
-          [[ "$ABX_DL_DIGEST" == sha256:* ]]
-          ABX_DL_IMAGE="${ABX_DL_REFERENCE%%@*}@${ABX_DL_DIGEST}"
-          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        if: inputs.push_digests
+      - name: Docker Login
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        if: inputs.push_digests
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Collect Docker labels
+      - name: Collect Docker tags
+        # https://github.com/docker/metadata-action
         id: docker_meta
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import tomllib
-          from pathlib import Path
+        uses: docker/metadata-action@v5
+        with:
+          images: archivebox/archivebox,nikisweeting/archivebox
+          tags: |
+              # :stable
+              type=ref,event=branch
+              # :0.7.3
+              type=semver,pattern={{version}}
+              # :0.7
+              type=semver,pattern={{major}}.{{minor}}
+              # :sha-463ea54
+              type=sha
+              # :latest
+              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
 
-          print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])
-          PY
-          )"
-          test -n "$VERSION"
-
-          {
-            echo 'labels<<EOF'
-            echo "org.opencontainers.image.version=${VERSION}"
-            echo "org.opencontainers.image.revision=${{ inputs.source_ref || github.sha }}"
-            echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
-            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
-            echo 'EOF'
-            echo "version=${VERSION}"
-          } >> "$GITHUB_OUTPUT"
-
-          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
-
-      - name: Build and push digest
+      - name: Build and push
         id: docker_build
-        if: inputs.push_digests
-        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
+        uses: docker/build-push-action@v5
         with:
           context: ./
           file: ./Dockerfile
           builder: ${{ steps.buildx.outputs.name }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-          build-args: |
-            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
-          pull: true
-          platforms: ${{ matrix.platform }}
-          # Keep the released image as the one cache authority. A second GHA
-          # cache can hold gzip variants of identical layers, and BuildKit may
-          # prefer those over :dev's zstd blobs. That counter-intuitively makes
-          # a cache hit larger than the released image. :dev advances only
-          # after both architectures pass the release checks.
-          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
-          # Inline metadata reuses the released compressed blobs directly;
-          # force-compression would recompress the whole ~740 MiB image. The
-          # compressed-size gate below remains the regression guard.
-          cache-to: |
-            ${{ inputs.push_digests && 'type=inline' || '' }}
-          outputs: |
-            ${{ inputs.full_tests && format('type=oci,dest={0}/archivebox-image.tar,compression=zstd,compression-level=3', runner.temp) || '' }}
-            ${{ inputs.push_digests && format('type=image,"name={0},{1}",push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=3', env.DOCKERHUB_IMAGE, env.GHCR_IMAGE) || '' }}
-
-      - name: Build pull request image
-        id: docker_build_pr
-        if: ${{ !inputs.push_digests }}
-        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
-        with:
-          context: ./
-          file: ./Dockerfile
-          builder: ${{ steps.buildx.outputs.name }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-          build-args: |
-            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
-          pull: true
-          platforms: ${{ matrix.platform }}
-          # Test the same layer lineage users receive. Reusing :dev avoids a
-          # second cache whose compression can drift from the released image.
-          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
-          outputs: |
-            type=docker,name=${{ matrix.local_tag }}
-            type=oci,dest=${{ runner.temp }}/archivebox-image.tar
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+          platforms: linux/amd64,linux/arm64
 
       - name: Image digest
-        if: inputs.push_digests
         run: echo ${{ steps.docker_build.outputs.digest }}
 
-      - name: Verify OpenCode install with a read-only browser profile
-        shell: bash
-        env:
-          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: |
-          set -Eeuo pipefail
-          PROFILE_DIR="$(mktemp -d)"
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' \
-            --volume "$PROFILE_DIR:/home/archivebox/.config/chromium:ro" \
-            "$IMAGE" bash -c '
-              set -Eeuo pipefail
-              test -w "$XDG_CONFIG_HOME"
-              test ! -w "$XDG_CONFIG_HOME/chromium"
-              # Check the shipped executable before the installer can repair
-              # an image that accidentally omitted this opt-in dependency.
-              "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
-              archivebox install opencode
-              "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
-            '
-
-      - name: Validate exact built image version and commit
-        if: inputs.full_tests
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}"
-          SHORT_SHA="${{ (inputs.source_ref || github.sha) }}"
-          SHORT_SHA="${SHORT_SHA::7}"
-
-          DATA_DIR="$(mktemp -d)"
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
-          ARCHIVEBOX_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version)"
-          printf '%s\n' "$ARCHIVEBOX_VERSION_OUTPUT"
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
-          IMAGE_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt)"
-          ARCHIVEBOX_VERSION_OUTPUT="$ARCHIVEBOX_VERSION_OUTPUT" IMAGE_VERSION_OUTPUT="$IMAGE_VERSION_OUTPUT" \
-            "$UV_BINARY" run --no-cache --no-project python - \
-            "ArchiveBox v${{ steps.docker_meta.outputs.version }}" \
-            "COMMIT_HASH=${SHORT_SHA}" \
-            "COMMIT_HASH=${GITHUB_SHA}" <<'PY'
-          import os
-          import sys
-
-          version_output = os.environ["ARCHIVEBOX_VERSION_OUTPUT"]
-          image_version_lines = os.environ["IMAGE_VERSION_OUTPUT"].splitlines()
-          if sys.argv[1] not in version_output:
-              raise SystemExit(f"Missing version marker: {sys.argv[1]}")
-          if sys.argv[2] not in version_output:
-              raise SystemExit(f"Missing short commit marker: {sys.argv[2]}")
-          if sys.argv[3] not in image_version_lines:
-              raise SystemExit(f"Missing exact commit marker: {sys.argv[3]}")
-          PY
-
-          "$DOCKER_BINARY" run --rm --user 911:911 --platform '${{ matrix.platform }}' --entrypoint /venv/bin/python3 "$IMAGE" -c \
-            'import json, pathlib, stat; cache_file = "/opt/archivebox/lib/env/derived.env"; raw = open(cache_file).read().strip(); records = json.loads(raw[len("ABXPKG_DERIVED_CACHE=\x27"):-1]); projections = [projection for record in records.values() for projection in record.get("request_exec_projections", {}).values()]; plans = [projection["validation"] for projection in projections]; fingerprints = [fingerprint for plan in plans for fingerprint in plan["fingerprint"]]; actual = lambda fingerprint: (lambda path, value: {"path": str(path.resolve(strict=False)), "size": value.st_size, "mtime_ns": value.st_mtime_ns, "mode": stat.S_IMODE(value.st_mode), "euid": value.st_uid})(pathlib.Path(fingerprint["path"]), pathlib.Path(fingerprint["path"]).stat()); mismatches = [(fingerprint, actual(fingerprint)) for fingerprint in fingerprints if fingerprint != actual(fingerprint)]; assert projections and {plan.get("euid") for plan in plans} == {911}; assert all(not record.get("script_exec_plans") for record in records.values()); assert "ABXPKG_TMP_CACHE_DIR" not in json.dumps(projections); assert not mismatches, mismatches[:3]'
-
-      - name: Validate compressed candidate size
-        if: inputs.push_digests
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          DIGEST='${{ steps.docker_build.outputs.digest }}'
-          [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
-          MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${DIGEST}")"
-          # Provenance wraps the real platform image and its attestation in an
-          # OCI index. Measure the image manifest, not the index/attestation.
-          IMAGE_DIGEST="$(MANIFEST="$MANIFEST" PLATFORM='${{ matrix.platform }}' "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); platform_os, platform_arch = os.environ["PLATFORM"].split("/")[:2]; print(next((item["digest"] for item in manifest.get("manifests", []) if item.get("platform", {}).get("os") == platform_os and item.get("platform", {}).get("architecture") == platform_arch), ""))')"
-          if [[ -n "$IMAGE_DIGEST" ]]; then
-            MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${IMAGE_DIGEST}")"
-          fi
-          COMPRESSED_SIZE="$(MANIFEST="$MANIFEST" "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); print(manifest["config"]["size"] + sum(layer["size"] for layer in manifest["layers"]))')"
-          LIMIT=$((1025 * 1024 * 1024))
-          echo "${DIGEST} compressed_size=$((COMPRESSED_SIZE / 1024 / 1024)) MiB limit=1025 MiB"
-          (( COMPRESSED_SIZE < LIMIT ))
-
-      - name: Resolve real Docker CLI test dependencies through abxpkg
-        id: mount_dependencies
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mount_env="$(
-            uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:mount_validation_binaries"
-          )"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$mount_env" >> "$GITHUB_ENV"
-
-          for binary_name in bash chmod curl exportfs mkdir mount.cifs mount.nfs smbd sudo systemctl umount; do
-            binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
-            test -L "$binary"
-            test -x "$binary"
-          done
-
-      - name: Diagnose failed mount dependency installation
-        if: failure() && steps.mount_dependencies.conclusion == 'failure'
-        shell: bash
-        run: |
-          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
-          sudo journalctl --since '-10 minutes' --no-pager \
-            -u smbd -u nmbd -u samba-ad-dc -u nfs-server
-          ps -eo pid,ppid,stat,etimes,wchan:32,comm
-
-      - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          NFS_SOURCE="/srv/archivebox-ci-nfs"
-          SMB_SOURCE="/srv/archivebox-ci-smb"
-          NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
-          SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
-          mounted_nfs=0
-          mounted_smb=0
-
-          cleanup() {
-            local original_status=$? cleanup_status=0
-            trap - EXIT
-            if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
-              cleanup_status=1
-            fi
-            if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
-              cleanup_status=1
-            fi
-            if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
-              cleanup_status=1
-            fi
-            if [[ "$original_status" != "0" ]]; then
-              exit "$original_status"
-            fi
-            exit "$cleanup_status"
-          }
-          trap cleanup EXIT
-
-          "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-          "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
-          "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-          "$SUDO_BINARY" "$BASH_BINARY" -c \
-            "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
-          "$SUDO_BINARY" "$BASH_BINARY" -c \
-            "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
-
-          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
-          "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
-          "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
-          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
-          "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
-
-          "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
-          mounted_nfs=1
-          "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
-            -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
-          mounted_smb=1
-
-          NFS_TEST_DIR="$NFS_MOUNT" \
-          SMB_TEST_DIR="$SMB_MOUNT" \
-          IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
-            "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
-
-      - name: Validate Docker port compatibility
-        shell: bash
-        env:
-          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
-
-      - name: Validate Docker CLI add/list and compose behavior
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        env:
-          ARCHIVEBOX_IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: |
-          set -Eeuo pipefail
-          "$MKDIR_BINARY" "${{ github.workspace }}/data"
-          "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
-
-          "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
-          "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
-
-          "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
-          echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add
-
-          archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
-          [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
-          [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
-
-          "$DOCKER_BINARY" compose run --rm archivebox init
-          "$DOCKER_BINARY" compose up -d --wait
-          "$DOCKER_BINARY" network connect archivebox_default www-nginx
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
-          [[ "$index_html" == *ArchiveBox* ]]
-          [[ "$admin_js" == *window.django* ]]
-
-          "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
-          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-          [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
-          "$DOCKER_BINARY" compose down
-
-      - name: Validate compressed image size
-        if: inputs.full_tests && !inputs.push_digests
-        env:
-          BUILD_METADATA: ${{ steps.docker_build_pr.outputs.metadata }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          LOCAL_IMAGE_ID="$("$DOCKER_BINARY" image inspect '${{ matrix.local_tag }}' --format '{{.Id}}')"
-          "$UV_BINARY" run --no-cache --no-project python - \
-            '${{ runner.temp }}/archivebox-image.tar' \
-            '${{ steps.docker_build_pr.outputs.digest }}' \
-            "$LOCAL_IMAGE_ID" \
-            "$((1025 * 1024 * 1024))" <<'PY'
-          from hashlib import sha256
-          import json
-          import os
-          import sys
-          import tarfile
-
-          archive_path, pushed_digest, local_image_id, limit_text = sys.argv[1:]
-          limit = int(limit_text)
-          metadata = json.loads(os.environ["BUILD_METADATA"])
-          if metadata["containerimage.digest"] != pushed_digest:
-              raise SystemExit("Build push transaction digest does not match the action output")
-          if not pushed_digest.startswith("sha256:") or len(pushed_digest) != 71:
-              raise SystemExit(f"Invalid pushed digest: {pushed_digest}")
-
-          with tarfile.open(archive_path) as archive:
-              def read_blob(descriptor):
-                  algorithm, digest = descriptor["digest"].split(":", 1)
-                  if algorithm != "sha256":
-                      raise SystemExit(f"Unsupported OCI digest: {descriptor['digest']}")
-                  member = archive.extractfile(f"blobs/sha256/{digest}")
-                  if member is None:
-                      raise SystemExit(f"Missing OCI blob: {descriptor['digest']}")
-                  blob = member.read()
-                  if len(blob) != descriptor["size"]:
-                      raise SystemExit(f"OCI size mismatch: {descriptor['digest']}")
-                  if sha256(blob).hexdigest() != digest:
-                      raise SystemExit(f"OCI digest mismatch: {descriptor['digest']}")
-                  return blob
-
-              index_member = archive.extractfile("index.json")
-              if index_member is None:
-                  raise SystemExit("OCI export is missing index.json")
-              index = json.load(index_member)
-
-              manifests = []
-
-              def collect(descriptor):
-                  document = json.loads(read_blob(descriptor))
-                  media_type = descriptor["mediaType"]
-                  if "image.index" in media_type or "manifest.list" in media_type:
-                      for child in document["manifests"]:
-                          collect(child)
-                  elif "image.manifest" in media_type:
-                      config_type = document["config"]["mediaType"]
-                      if "image.config" in config_type or "container.image" in config_type:
-                          manifests.append(document)
-
-              for root in index["manifests"]:
-                  collect(root)
-              if len(manifests) != 1:
-                  raise SystemExit(f"Expected one runnable image manifest, found {len(manifests)}")
-
-              manifest = manifests[0]
-              config_digest = manifest["config"]["digest"]
-              if metadata["containerimage.config.digest"] != config_digest:
-                  raise SystemExit("OCI config digest does not match the pushed build result")
-              if local_image_id != config_digest:
-                  raise SystemExit("Locally tested image does not match the OCI build result")
-              descriptors = [manifest["config"], *manifest["layers"]]
-              for descriptor in descriptors:
-                  read_blob(descriptor)
-              total = sum(descriptor["size"] for descriptor in descriptors)
-
-          print(
-              f"{pushed_digest} compressed_size={total // 1024 // 1024} MiB "
-              f"limit={limit // 1024 // 1024} MiB",
-          )
-          if total > limit:
-              raise SystemExit(f"{pushed_digest} is over the compressed size limit")
-          PY
-
-      - name: Export digest
-        if: inputs.push_digests
-        env:
-          ABX_DL_IMAGE_REF: ${{ steps.abx_dl_image.outputs.image }}
-          ARTIFACT_NAME: ${{ matrix.artifact_name }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mkdir -p /tmp/digests
-          digest="${{ steps.docker_build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-          printf '%s\n' '${{ inputs.source_ref || github.sha }}' > /tmp/digests/source-ref
-          printf '%s\n' "$ABX_DL_IMAGE_REF" > "/tmp/digests/abx-dl-image-$ARTIFACT_NAME"
-
-      - name: Upload digest
-        if: inputs.push_digests
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+      - name: Update README
+        uses: peter-evans/dockerhub-description@v4
         with:
-          name: ${{ matrix.artifact_name }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: archivebox/archivebox
+
+      # This ugly bit is necessary if you don't want your cache to grow forever
+      # until it hits GitHub's limit of 5GB.
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/workflows/docs-links.yml

@@ -1,38 +0,0 @@
-name: Documentation link checks
-
-on:
-  schedule:
-    - cron: '17 10 * * *' # Daily at 10:17 UTC, away from the busy top of the hour.
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-env:
-  PYTHONIOENCODING: utf-8
-
-jobs:
-  links:
-    name: Documentation links and anchors
-    runs-on: ubuntu-24.04
-    timeout-minutes: 5
-    permissions:
-      contents: read
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-      - name: List tracked documentation and source inputs
-        run: |
-          git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > "$RUNNER_TEMP/link-inputs.txt"
-      - uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
-        with:
-          lycheeVersion: v0.24.2
-          args: >-
-            --config .github/lychee.toml
-            --root-dir "${{ github.workspace }}"
-            --files-from "${{ runner.temp }}/link-inputs.txt"
-          fail: true
-          failIfEmpty: true

.github/workflows/docs.yml

@@ -1,195 +0,0 @@
-name: Documentation tests
-
-on:
-  schedule:
-    - cron: '7 5,17 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-env:
-  PYTHONIOENCODING: utf-8
-  USE_COLOR: "False"
-  SHOW_PROGRESS: "False"
-
-jobs:
-  changes:
-    runs-on: ubuntu-24.04
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      key: ${{ steps.check.outputs.key }}
-    steps:
-      - id: check
-        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
-
-  discover:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    name: Discover documentation examples
-    runs-on: ubuntu-24.04
-    timeout-minutes: 5
-    outputs:
-      matrix: ${{ steps.inventory.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-      - name: Install ArchiveBox development dependencies
-        run: uv sync --no-cache --locked --dev
-      - name: Resolve documentation tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
-        run: |
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          uv run --no-cache --no-sync abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries"
-          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
-          mmdc_binary="$ABXPKG_LIB_DIR/env/bin/mmdc"
-          nginx_binary="$ABXPKG_LIB_DIR/env/bin/nginx"
-          test -L "$bash_binary"
-          test -L "$mmdc_binary"
-          test -L "$nginx_binary"
-          test -x "$bash_binary"
-          test -x "$mmdc_binary"
-          test -x "$nginx_binary"
-          {
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-            echo "BASH_BINARY=$bash_binary"
-          } >> "$GITHUB_ENV"
-      - id: inventory
-        name: Validate the complete authored documentation inventory
-        run: |
-          matrix="$(uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py matrix)"
-          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
-
-  examples:
-    name: docs/${{ matrix.name }}
-    needs: discover
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 15
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        with:
-          python-version: "3.13"
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-      - name: Install ArchiveBox
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        run: uv sync --no-cache --locked --dev
-      - name: Resolve documentation runtime through abxpkg
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
-        run: |
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          dependency_sources=(
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
-          )
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
-          fi
-          if [[ '${{ matrix.name }}' == ubuntu-AGENTS || '${{ matrix.name }}' == ubuntu-skills-archivebox-SKILL || '${{ matrix.name }}' == ubuntu-docs-Configuration || '${{ matrix.name }}' == ubuntu-docs-Setting-up-Authentication ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          fi
-          if [[ '${{ matrix.environment }}' == docker ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
-          elif [[ '${{ matrix.environment }}' == macos ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")
-          fi
-          uv run --no-cache --no-sync abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            "${dependency_sources[@]}"
-          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
-          python_binary="$ABXPKG_LIB_DIR/env/bin/python"
-          uv_binary="$ABXPKG_LIB_DIR/env/bin/uv"
-          test -L "$bash_binary"
-          test -x "$bash_binary"
-          test -L "$python_binary"
-          test -x "$python_binary"
-          test -L "$uv_binary"
-          test -x "$uv_binary"
-          {
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-            echo "BASH_BINARY=$bash_binary"
-            echo "PYTHON_BINARY=$python_binary"
-            echo "UV_BINARY=$uv_binary"
-          } >> "$GITHUB_ENV"
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            sudo_binary="$ABXPKG_LIB_DIR/env/bin/sudo"
-            test -L "$sudo_binary"
-            test -x "$sudo_binary"
-            echo "SUDO_BINARY=$sudo_binary" >> "$GITHUB_ENV"
-          fi
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-          fi
-          if [[ '${{ matrix.environment }}' == docker ]]; then
-            docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
-            test -L "$docker_binary"
-            test -x "$docker_binary"
-            echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV"
-          elif [[ '${{ matrix.environment }}' == macos ]]; then
-            test -L "$ABXPKG_LIB_DIR/env/bin/brew"
-            test -x "$ABXPKG_LIB_DIR/env/bin/brew"
-          fi
-      - name: Build the exact checkout's documentation image
-        if: ${{ matrix.environment == 'docker' }}
-        run: |
-          ABX_DL_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(dep.removeprefix("abx-dl==") for dep in tomllib.load(open("pyproject.toml", "rb"))["project"]["dependencies"] if dep.startswith("abx-dl==")))')"
-          "$DOCKER_BINARY" build --build-arg "ABX_DL_IMAGE=archivebox/abx-dl:${ABX_DL_VERSION}" --tag archivebox/archivebox:dev .
-      - name: Run the Ubuntu, Docker, or macOS documentation lane
-        if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
-        run: uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
-      - name: Run the root documentation lane
-        if: ${{ matrix.environment == 'root' }}
-        run: |
-          "$SUDO_BINARY" \
-            --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
-            "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
-      - name: Diagnose failed Linux package installation
-        if: failure() && runner.os == 'Linux'
-        run: |
-          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
-          apt-config dump Acquire::Retries Acquire::http::Timeout Acquire::https::Timeout
-          ps -eo pid,ppid,stat,etimes,wchan:32,comm
-
-  complete:
-    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
-    needs: [changes, discover, examples]
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - run: echo 'Scheduled work completed successfully.'

.github/workflows/homebrew.yml

@@ -0,0 +1,51 @@
+name: Build Homebrew package
+
+on:
+  workflow_dispatch:
+  push:
+
+
+jobs:
+  build:
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      # TODO: modify archivebox.rb to update src url, hashes, and dependencies
+
+      - name: Build Homebrew Bottle
+        run: |
+          pip3 install --upgrade pip setuptools wheel
+          cd brew_dist/
+          brew install --build-bottle ./archivebox.rb
+          # brew bottle archivebox
+          archivebox version
+
+      - name: Add some links to test
+        run: |
+          mkdir data && cd data
+          archivebox init --setup
+          archivebox add 'https://example.com'
+          archivebox version
+          archivebox status
+
+      # - name: Commit built package
+      #   run: |
+      #     cd brew_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Homebrew package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/homebrew-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: brew_dist
+
+      # TODO: push bottle homebrew core PR with latest changes

.github/workflows/jekyll-gh-pages.yml

@@ -0,0 +1,58 @@
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Build GitHub Pages website
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["dev"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Copy README.md into place
+        run: |
+          rm ./website/README.md
+          cp ./README.md ./website/README.md
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

.github/workflows/lint.yml

@@ -1,38 +1,35 @@
 name: Run linters
 
 on:
-  workflow_call:
+  workflow_dispatch:
+  push:
 
 env:
-  UV_NO_SOURCES: "1"
+  MAX_LINE_LENGTH: 110
 
 jobs:
   lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v2
         with:
           submodules: true
           fetch-depth: 1
 
       - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/setup-python@v1
         with:
-          python-version: "3.13"
+          python-version: 3.9
           architecture: x64
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+      - name: Install flake8
+        run: |
+          pip install flake8
 
-      - name: Verify checked dependency lock
-        run: uv lock --no-cache --check
-
-      - name: Install dependencies with uv
-        run: uv sync --no-cache --locked --dev --extra sonic --extra debug
-
-      - name: Run prek
-        run: uv run --no-cache --no-sync --no-sources prek run --all-files
+      - name: Lint with flake8
+        run: |
+          cd archivebox
+          # one pass for show-stopper syntax errors or undefined names
+          flake8 . --count --show-source --statistics
+          # one pass for small stylistic things
+          flake8 . --count --max-line-length="$MAX_LINE_LENGTH" --statistics

.github/workflows/macos.yml

@@ -1,48 +0,0 @@
-name: macOS compatibility
-
-on:
-  schedule:
-    - cron: '51 2,10,18 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-jobs:
-  changes:
-    runs-on: ubuntu-24.04
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      key: ${{ steps.check.outputs.key }}
-    steps:
-      - id: check
-        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
-
-  install-cli:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    uses: ./.github/workflows/test.yml
-    with:
-      platform: macos
-    secrets: inherit
-
-  python-artifacts:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    uses: ./.github/workflows/pip.yml
-    with:
-      platforms: '["macos-15"]'
-    secrets: inherit
-
-  complete:
-    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
-    needs: [changes, install-cli, python-artifacts]
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - run: echo 'Scheduled macOS coverage completed successfully.'

.github/workflows/pip.yml

@@ -1,248 +1,78 @@
 name: Build Pip package
 
 on:
-  workflow_call:
-    inputs:
-      platforms:
-        description: Runner matrix for package compatibility
-        required: false
-        default: '["ubuntu-24.04"]'
-        type: string
-      full_tests:
-        description: Run the full package compatibility matrix instead of one release smoke check
-        required: false
-        default: true
-        type: boolean
+  workflow_dispatch:
+  push:
 
-env:
-  UV_VERSION: "0.11.3"
 
 jobs:
   build:
-    name: build distributions
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+    runs-on: ubuntu-22.04
 
-      - name: Set up Python 3.13
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-          architecture: x64
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: ${{ env.UV_VERSION }}
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Build every distribution
-        run: |
-          set -Eeuo pipefail
-          uv build --no-cache --all-packages --no-sources --out-dir dist --clear
-
-          shopt -s nullglob
-          artifacts=(dist/*)
-          wheels=(dist/archivebox-*.whl)
-          sdists=(dist/archivebox-*.tar.gz)
-          [[ "${#artifacts[@]}" -gt 0 ]]
-          [[ "${#wheels[@]}" -eq 1 ]]
-          [[ "${#sdists[@]}" -eq 1 ]]
-          printf '%s\n' "$GITHUB_SHA" > dist/COMMIT_SHA
-          uv run --no-cache --no-project python - <<'PY'
-          import hashlib
-          from pathlib import Path
-
-          root = Path("dist")
-          wheels = list(root.glob("archivebox-*.whl"))
-          sdists = list(root.glob("archivebox-*.tar.gz"))
-          if len(wheels) != 1 or len(sdists) != 1:
-              raise SystemExit("Expected exactly one ArchiveBox wheel and one sdist")
-          artifacts = [root / "COMMIT_SHA", wheels[0], sdists[0]]
-          checksum_lines = []
-          for artifact in artifacts:
-              digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
-              line = f"{digest}  {artifact.name}"
-              checksum_lines.append(line)
-              print(line)
-          Path("dist/SHA256SUMS").write_text("\n".join(checksum_lines) + "\n")
-          PY
-
-      - name: Release wheel import and CLI smoke
-        if: ${{ !inputs.full_tests }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          shopt -s nullglob
-          wheels=(dist/archivebox-*.whl)
-          [[ "${#wheels[@]}" -eq 1 ]]
-          uv sync --locked --no-cache --no-sources --no-dev --no-install-project
-          uv pip install --no-cache --no-deps --python .venv/bin/python "${wheels[0]}"
-          unset PYTHONPATH
-          cd "$(mktemp -d)"
-          VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project python -c 'import archivebox'
-          VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project archivebox version
-
-      - name: Upload every distribution
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: python-distributions
-          path: dist/*
-          if-no-files-found: error
-          retention-days: 7
-
-  install-smoke:
-    name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }}
-    if: inputs.full_tests
-    needs: build
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ${{ fromJSON(inputs.platforms) }}
-        python: ["3.13", "3.14"]
-        artifact: [wheel, sdist]
+    permissions:
+      id-token: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
         with:
+          submodules: true
           fetch-depth: 1
 
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+      - name: Set up Python
+        uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python }}
+          python-version: '3.11'
+          architecture: x64
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+      - uses: awalsh128/cache-apt-pkgs-action@latest
         with:
-          version: ${{ env.UV_VERSION }}
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
 
-      - name: Download distributions
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+      - uses: pdm-project/setup-pdm@v3
         with:
-          name: python-distributions
-          path: ${{ runner.temp }}/python-distributions
+          python-version: '3.11'
+          cache: true
 
-      - name: Install and smoke-test the built ${{ matrix.artifact }}
-        shell: bash
+      - name: Install dependencies
+        run: pdm install --fail-fast --no-lock --dev --group=':all' --no-self
+
+      - name: Build package
         run: |
-          set -Eeuo pipefail
-          shopt -s nullglob
+          pdm build
 
-          case "${{ matrix.artifact }}" in
-            wheel) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.whl) ;;
-            sdist) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.tar.gz) ;;
-            *) exit 2 ;;
-          esac
-          [[ "${#candidates[@]}" -eq 1 ]]
-          artifact="${candidates[0]}"
+      - name: Install from build
+        run: pip install ./dist/archivebox-*.whl
 
-          smoke_root="$(mktemp -d "$RUNNER_TEMP/archivebox-package-smoke.XXXXXX")"
-          trap 'rm -rf "$smoke_root"' EXIT
-          smoke_env="$smoke_root/env"
-          data_dir="$smoke_root/data"
-          mkdir -p "$data_dir"
+      - name: Add some links to test
+        run: |
+          mkdir data && cd data
+          archivebox init
+          archivebox add 'https://example.com'
+          archivebox version
+          archivebox status
 
-          dependency_artifacts=()
-          while IFS= read -r dependency_artifact; do
-            dependency_artifacts+=("$dependency_artifact")
-          done < <(uv run --no-cache --no-project python - <<'PY'
-          import tomllib
+      #- name: Publish package distributions to PyPI
+      #  run: pdm publish --no-build
 
-          locked = {
-              package["name"]: package
-              for package in tomllib.load(open("uv.lock", "rb"))["package"]
-          }
-          for package_name in ("abx-dl", "abx-plugins", "abxpkg", "abxbus"):
-              package = locked[package_name]
-              wheels = package.get("wheels") or []
-              if len(wheels) != 1:
-                  raise SystemExit(f"Expected exactly one locked wheel for {package_name}")
-              print(wheels[0]["url"])
-          PY
-          )
+      # - name: Push build to PyPI
+      #   run: |
+      #     cd pip_dist/
+      #     python3 -m twine upload --repository testpypi pip_dist/*.{whl,tar.gz}
+      #     python3 -m twine upload --repository pypi pip_dist/*.{whl,tar.gz}
 
-          uv venv --python "${{ matrix.python }}" "$smoke_env"
-          uv pip install --no-cache \
-            --python "$smoke_env" \
-            "${dependency_artifacts[@]}" \
-            "$artifact"
+      # - name: Commit built package
+      #   run: |
+      #     cd pip_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Pip package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/pip-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: pip_dist
 
-          unset PYTHONPATH
-          cd "$data_dir"
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync python - <<'PY'
-          import os
-          from pathlib import Path
-
-          import archivebox
-
-          installed_path = Path(archivebox.__file__).resolve()
-          environment_path = Path(os.environ["VIRTUAL_ENV"]).resolve()
-          workspace_path = Path(os.environ["GITHUB_WORKSPACE"]).resolve()
-          assert installed_path.is_relative_to(environment_path), (installed_path, environment_path)
-          assert not installed_path.is_relative_to(workspace_path), (installed_path, workspace_path)
-          print(installed_path)
-          PY
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox version
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox init
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox status
-
-          if [[ "${{ runner.os }}" == "Linux" ]]; then
-            root_tool_dir="/root/archivebox-package-smoke-${{ matrix.artifact }}-${{ matrix.python }}"
-            uv_binary="$(command -v uv)"
-            sudo rm -rf "$root_tool_dir"
-            sudo mkdir -p "$root_tool_dir/data"
-
-            root_install_args=()
-            for dependency_artifact in "${dependency_artifacts[@]}"; do
-              root_install_args+=(--with "$dependency_artifact")
-            done
-            sudo UV_TOOL_DIR="$root_tool_dir/tools" UV_TOOL_BIN_DIR="$root_tool_dir/bin" \
-              "$uv_binary" tool install --no-cache --python "${{ matrix.python }}" \
-              "${root_install_args[@]}" "$artifact"
-
-            sudo useradd --system --create-home --home-dir /var/lib/archivebox --shell /bin/bash archivebox 2>/dev/null || true
-            sudo rm -rf /var/lib/archivebox
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data' && archivebox init"
-            sudo stat -c %U /var/lib/archivebox | grep -qx archivebox
-            sudo stat -c %U "$root_tool_dir/data" | grep -qx archivebox
-            if sudo find "$root_tool_dir/data" -xdev -user root -print -quit | grep -q .; then
-              echo "Root-owned files remain in $root_tool_dir/data" >&2
-              exit 1
-            fi
-
-            sudo mkdir "$root_tool_dir/external"
-            sudo ln -s "$root_tool_dir/external" "$root_tool_dir/data/external-link"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data' && archivebox status"
-            sudo stat -c %U "$root_tool_dir/external" | grep -qx root
-
-            sudo mkdir "$root_tool_dir/data-universal-init"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-universal-init' && archivebox status --init"
-            sudo stat -c %U "$root_tool_dir/data-universal-init" | grep -qx archivebox
-
-            sudo mkdir "$root_tool_dir/data-install"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-install' && archivebox install --dry-run"
-            sudo stat -c %U "$root_tool_dir/data-install" | grep -qx archivebox
-
-            sudo useradd --create-home archivebox-smoke-user
-            smoke_user_group="$(id -gn archivebox-smoke-user)"
-            sudo mkdir "$root_tool_dir/data-mixed-owner"
-            sudo chown archivebox-smoke-user:root "$root_tool_dir/data-mixed-owner"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-mixed-owner' && archivebox init"
-            sudo stat -c %U:%G "$root_tool_dir/data-mixed-owner" | grep -qx "archivebox-smoke-user:$smoke_user_group"
-            if sudo find "$root_tool_dir/data-mixed-owner" -xdev -user root -print -quit | grep -q .; then
-              echo "Root-owned files remain in $root_tool_dir/data-mixed-owner" >&2
-              exit 1
-            fi
-            sudo rm -rf "$root_tool_dir"
-          fi

.github/workflows/prepare-release.yml

@@ -1,55 +0,0 @@
-name: Prepare release version
-
-on:
-  workflow_call:
-    outputs:
-      run_jobs:
-        description: Whether this revision owns its release version and should run expensive jobs
-        value: ${{ jobs.prepare.outputs.run_jobs }}
-
-permissions:
-  contents: write
-
-jobs:
-  prepare:
-    name: Prepare the version before starting expensive jobs
-    permissions:
-      contents: write
-    runs-on: ubuntu-24.04
-    outputs:
-      run_jobs: ${{ github.event_name != 'push' || steps.channel.outputs.stable == 'true' || steps.version.outputs.action == 'ready' || steps.version.outputs.action == 'reserved' }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          token: ${{ github.event_name == 'push' && secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - name: Keep synchronized stable branches on the released version
-        id: channel
-        if: github.event_name == 'push'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
-          git fetch --no-tags origin main
-          MAIN_SHA="$(git rev-parse origin/main)"
-          # Stable reconciliation may preserve dev history with a merge commit.
-          # Its tree still matches main; bumping it would undo synchronization
-          # before the stable publisher can advance the Docker :dev tag.
-          if [[ "$GITHUB_REF_NAME" == dev && "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
-             git merge-base --is-ancestor "$MAIN_SHA" "$GITHUB_SHA" &&
-             git diff --quiet "$MAIN_SHA" "$GITHUB_SHA"; then
-            echo 'stable=true' >> "$GITHUB_OUTPUT"
-          else
-            echo 'stable=false' >> "$GITHUB_OUTPUT"
-          fi
-
-      - id: version
-        if: github.event_name == 'push' && steps.channel.outputs.stable != 'true'
-        uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
-        with:
-          package: archivebox
-          branch: ${{ github.ref_name }}
-          tag-prefix: v
-          version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}

.github/workflows/release.yml

@@ -1,584 +0,0 @@
-name: Release
-
-on:
-  workflow_run:
-    workflows: [CI]
-    types: [completed]
-    branches: [dev, main]
-  workflow_dispatch:
-    inputs:
-      release_branch:
-        description: Branch being promoted
-        required: true
-        type: choice
-        options: [main, dev]
-      release_sha:
-        description: Exact tested commit to release
-        required: true
-        type: string
-      ci_run_id:
-        description: Successful CI run containing the tested Python artifacts
-        required: true
-        type: string
-      docker_digest_run_id:
-        description: Manual Docker digest build run containing the tested image digests
-        required: true
-        type: string
-
-permissions: {}
-
-concurrency:
-  group: release-${{ github.repository }}-${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
-  cancel-in-progress: false
-
-env:
-  DOCKERHUB_IMAGE: archivebox/archivebox
-  GHCR_IMAGE: ghcr.io/archivebox/archivebox
-  RELEASE_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
-  RELEASE_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.release_sha || github.event.workflow_run.head_sha }}
-
-jobs:
-  candidate:
-    if: >-
-      github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.event == 'push' &&
-      (github.event.workflow_run.head_branch == 'dev' || github.event.workflow_run.head_branch == 'main') ||
-      (github.event_name == 'workflow_dispatch' && inputs.release_branch == 'main')
-    runs-on: ubuntu-24.04
-    permissions:
-      actions: read
-      contents: read
-    outputs:
-      ready: ${{ steps.verified.outputs.ready }}
-      artifact_run_id: ${{ steps.verified.outputs.artifact_run_id }}
-      digest_run_id: ${{ steps.verified.outputs.digest_run_id }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          ref: ${{ env.RELEASE_SHA }}
-      - id: owner
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          BRANCH_SHA="$(git ls-remote origin "refs/heads/$RELEASE_BRANCH" | cut -f1)"
-          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && "$BRANCH_SHA" != "$RELEASE_SHA" ]]; then
-            echo 'ready=false' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
-              echo "Refusing to publish non-stable version $VERSION as a main release" >&2
-              exit 1
-            }
-            git fetch --no-tags origin main
-            git merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
-              echo "Refusing to publish $RELEASE_SHA: it is not on main" >&2
-              exit 1
-            }
-            TAG_TARGET="$(git ls-remote origin "refs/tags/v$VERSION" | cut -f1)"
-            if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && -n "$TAG_TARGET" && "$TAG_TARGET" != "$RELEASE_SHA" ]]; then
-              echo 'ready=false' >> "$GITHUB_OUTPUT"
-              exit 0
-            fi
-            echo 'ready=true' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          # A stable release fast-forwards dev to main. Only main publishes it;
-          # the resulting dev workflow is not a new release candidate.
-          if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]]; then
-            echo 'ready=false' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          TARGET="$(git ls-remote origin "refs/tags/release-candidate/$VERSION" | awk '{print $1}')"
-          [[ "$TARGET" == "$RELEASE_SHA" ]] && echo 'ready=true' >> "$GITHUB_OUTPUT" || echo 'ready=false' >> "$GITHUB_OUTPUT"
-
-      - name: Require successful full CI for this commit
-        id: verified
-        if: steps.owner.outputs.ready == 'true'
-        env:
-          GH_TOKEN: ${{ github.token }}
-          MANUAL_CI_RUN_ID: ${{ inputs.ci_run_id }}
-          MANUAL_DIGEST_RUN_ID: ${{ inputs.docker_digest_run_id }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then
-            CI_RUN_ID="$MANUAL_CI_RUN_ID"
-            DIGEST_RUN_ID="$MANUAL_DIGEST_RUN_ID"
-            CI_STATUS="$(gh run view "$CI_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status,conclusion,headSha,event,name \
-              --jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
-            [[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
-          else
-            CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
-              --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
-              --json databaseId,status,conclusion \
-              --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
-            [[ -n "$CI_RUN_ID" ]] || exit 0
-            DIGEST_RUN_ID="$CI_RUN_ID"
-          fi
-          ARTIFACT_RUN_ID="$CI_RUN_ID"
-          {
-            echo 'ready=true'
-            echo "artifact_run_id=$ARTIFACT_RUN_ID"
-            echo "digest_run_id=$DIGEST_RUN_ID"
-          } >> "$GITHUB_OUTPUT"
-
-  python-release:
-    name: Publish tested Python artifacts
-    needs: candidate
-    if: needs.candidate.outputs.ready == 'true'
-    runs-on: ubuntu-24.04
-    env:
-      CI_RUN_ID: ${{ needs.candidate.outputs.artifact_run_id }}
-      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
-    environment: pypi
-    permissions:
-      actions: read
-      contents: write
-      deployments: read
-      id-token: write
-    outputs:
-      version: ${{ steps.version.outputs.version }}
-      tag: ${{ steps.version.outputs.tag }}
-      release_ready: ${{ steps.release-state.outputs.ready }}
-    steps:
-      - name: Checkout the tested commit
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          ref: ${{ env.RELEASE_SHA }}
-          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      # This precedes every stable upload, including workflow_dispatch. A green
-      # build is not evidence that our automatically updated test servers ran it.
-      - name: Download tested Docker base-image metadata
-        if: env.RELEASE_BRANCH == 'main'
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          path: ${{ runner.temp }}/tested-docker-artifacts
-          pattern: digest-*
-          merge-multiple: true
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
-
-      - name: Require matching acceptance on Cabbage
-        if: env.RELEASE_BRANCH == 'main'
-        env:
-          GH_TOKEN: ${{ github.token }}
-          STAGING_DOCKER_ARTIFACT_DIR: ${{ runner.temp }}/tested-docker-artifacts
-        run: uv run --no-project python bin/verify-staging-release.py
-
-      - name: Resolve release binaries through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" >/dev/null
-          for name in uv gh git jq curl; do
-            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
-            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
-          done
-          {
-            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
-            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
-            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-            echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
-          } >> "$GITHUB_ENV"
-
-      - name: Download exact tested distributions
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: python-distributions
-          path: ${{ runner.temp }}/python-distributions
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.CI_RUN_ID }}
-
-      - name: Record release version
-        id: version
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$($UV_BINARY run --no-cache --no-project python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
-          echo "tag=v$VERSION" >> "$GITHUB_OUTPUT"
-
-      - name: Validate release branch and version
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION='${{ steps.version.outputs.version }}'
-          case "$RELEASE_BRANCH" in
-            main)
-              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
-                echo "Refusing to publish non-stable version $VERSION from main" >&2
-                exit 1
-              }
-              ;;
-            dev)
-              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]] || {
-                echo "Refusing to publish ArchiveBox $VERSION from dev; only archivebox:main may publish non-prerelease versions." >&2
-                exit 1
-              }
-              ;;
-            *)
-              echo "Refusing to publish ArchiveBox $VERSION from unsupported branch $RELEASE_BRANCH; only archivebox:dev and archivebox:main may publish." >&2
-              exit 1
-              ;;
-          esac
-
-      - name: Publish the exact tested distributions
-        env:
-          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-          RELEASE_DISTRIBUTIONS_DIR: ${{ runner.temp }}/python-distributions
-        run: ./bin/release.sh
-
-      - name: Confirm this tested commit owns the release tag
-        id: release-state
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          TARGET=""
-          while read -r sha _; do
-            TARGET="$sha"
-          done < <("$GIT_BINARY" ls-remote origin "refs/tags/${{ steps.version.outputs.tag }}")
-          if [[ "$TARGET" == "$RELEASE_SHA" ]]; then
-            echo "ready=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "ready=false" >> "$GITHUB_OUTPUT"
-          fi
-
-  docker-release:
-    name: Publish tested multiarch image
-    if: >-
-      (github.event_name == 'workflow_dispatch') ||
-      (github.event.workflow_run.conclusion == 'success' &&
-       github.event.workflow_run.event == 'push')
-    needs: [candidate, python-release]
-    runs-on: ubuntu-24.04
-    env:
-      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
-    permissions:
-      actions: read
-      contents: write
-      packages: write
-    steps:
-      - name: Checkout the tested commit
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          ref: ${{ env.RELEASE_SHA }}
-          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - name: Reconcile dev to the tested stable release
-        if: env.RELEASE_BRANCH == 'main' && needs.python-release.outputs.release_ready == 'true'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          git fetch --no-tags origin main dev
-          MAIN_SHA="$(git rev-parse origin/main)"
-          DEV_SHA="$(git rev-parse origin/dev)"
-          if [[ "$MAIN_SHA" != "$RELEASE_SHA" || "$DEV_SHA" == "$RELEASE_SHA" ]]; then
-            exit 0
-          fi
-          push_dev() {
-            if ! git push origin "$1:refs/heads/dev"; then
-              CURRENT_DEV="$(git ls-remote origin refs/heads/dev | cut -f1)"
-              if [[ "$CURRENT_DEV" == "$DEV_SHA" ]]; then
-                echo "Failed to push dev although it still points to $DEV_SHA" >&2
-                exit 1
-              fi
-              echo "Leaving dev at $CURRENT_DEV: it advanced during release reconciliation"
-            fi
-          }
-          if git merge-base --is-ancestor "$DEV_SHA" "$RELEASE_SHA"; then
-            push_dev "$RELEASE_SHA"
-            exit 0
-          else
-            STATUS=$?
-            [[ "$STATUS" -eq 1 ]] || exit "$STATUS"
-          fi
-          BASE_SHA="$(git merge-base "$MAIN_SHA" "$DEV_SHA")"
-          if git diff --quiet "$BASE_SHA" "$DEV_SHA" -- . ':!pyproject.toml' ':!uv.lock' ':!etc/package.json'; then
-            :
-          else
-            STATUS=$?
-            [[ "$STATUS" -eq 1 ]] || exit "$STATUS"
-            echo "Leaving dev at $DEV_SHA: it contains changes beyond release metadata"
-            exit 0
-          fi
-          normalized_metadata() {
-            case "$2" in
-              pyproject.toml)
-                git show "$1:$2" | awk '
-                  /^\[/ {section = $0}
-                  section == "[project]" && /^version = "[^"]+"$/ {print "version = \"__VERSION__\""; next}
-                  section == "[tool.bumpver]" && /^current_version = "[^"]+"$/ {print "current_version = \"__VERSION__\""; next}
-                  {print}
-                ' ;;
-              uv.lock)
-                git show "$1:$2" | awk 'previous == "name = \"archivebox\"" && /^version = / {print "version = \"__VERSION__\""; previous = $0; next} {print; previous = $0}' ;;
-              etc/package.json)
-                git show "$1:$2" | sed -E 's/^(  "version": )"[^"]+"(,?)$/\1"__VERSION__"\2/' ;;
-            esac
-          }
-          for path in pyproject.toml uv.lock etc/package.json; do
-            BASE_METADATA="$(normalized_metadata "$BASE_SHA" "$path")"
-            DEV_METADATA="$(normalized_metadata "$DEV_SHA" "$path")"
-            if [[ "$BASE_METADATA" != "$DEV_METADATA" ]]; then
-              echo "Leaving dev at $DEV_SHA: $path contains changes beyond release version metadata"
-              exit 0
-            fi
-          done
-          # A concurrent release bot may bump only dev's own version metadata.
-          # Keep that commit as a parent while preserving the exact tested stable tree.
-          git switch --detach "$MAIN_SHA"
-          git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \
-            merge -s ours --no-ff -m "Merge tested stable release into dev" "$DEV_SHA"
-          MERGE_SHA="$(git rev-parse HEAD)"
-          push_dev "$MERGE_SHA"
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      - name: Resolve release and Docker binaries through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
-          for name in uv gh git jq docker; do
-            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
-            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
-          done
-          {
-            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
-            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
-            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-            echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
-          } >> "$GITHUB_ENV"
-
-      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
-        with:
-          install: true
-
-      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ github.token }}
-
-      - name: Download exact tested image digests
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          path: /tmp/digests
-          pattern: digest-*
-          merge-multiple: true
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
-
-      - name: Verify exact successful digest build run
-        shell: bash
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          set -Eeuo pipefail
-          RUN_STATUS="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status --jq '.status')"
-          RUN_CONCLUSION="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json conclusion --jq '.conclusion')"
-          [[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]]
-          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
-            RUN_SHA="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha')"
-            [[ "$RUN_SHA" == "$RELEASE_SHA" ]]
-          fi
-
-      - name: Verify digest set and collect tags
-        id: docker_meta
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          [[ "${#DIGESTS[@]}" -eq 2 ]]
-          for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
-          [[ "$(< /tmp/digests/source-ref)" == "$RELEASE_SHA" ]]
-
-          VERSION='${{ needs.python-release.outputs.version }}'
-          SHORT_SHA="${RELEASE_SHA::12}"
-          TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}^{}")"
-          TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
-          if [[ -z "$TAG_TARGET" ]]; then
-            TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
-            TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
-          fi
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
-              echo "Refusing to tag :latest with prerelease $VERSION" >&2
-              exit 1
-            }
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] || {
-              echo "Refusing to tag :latest: v$VERSION does not point to $RELEASE_SHA" >&2
-              exit 1
-            }
-            $GIT_BINARY fetch --no-tags origin main
-            $GIT_BINARY merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
-              echo "Refusing to tag :latest: $RELEASE_SHA is not on main" >&2
-              exit 1
-            }
-            MAIN_VERSION="$($GIT_BINARY show origin/main:pyproject.toml | sed -nE 's/^version = "([^"]+)".*/\1/p' | head -n 1)"
-            [[ "$MAIN_VERSION" == "$VERSION" ]] || {
-              echo "Refusing to tag :latest: main is now $MAIN_VERSION, not $VERSION" >&2
-              exit 1
-            }
-          fi
-          SYNC_DEV=false
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            $GIT_BINARY fetch --no-tags origin main dev
-            MAIN_TARGET="$($GIT_BINARY rev-parse origin/main)"
-            DEV_TARGET="$($GIT_BINARY rev-parse origin/dev)"
-            # Recovery can add CI/docs fixes after the immutable release tag.
-            # Both branches and every shipped application file must agree.
-            if $GIT_BINARY merge-base --is-ancestor "$MAIN_TARGET" "$DEV_TARGET" &&
-               $GIT_BINARY diff --quiet "$MAIN_TARGET" "$DEV_TARGET" &&
-               $GIT_BINARY diff --quiet "$RELEASE_SHA" "$MAIN_TARGET" -- . ':!.github' ':!README.md'; then
-              SYNC_DEV=true
-            fi
-          fi
-          {
-            echo 'dockerhub_tags<<EOF'
-            if [[ "$RELEASE_BRANCH" == dev ]]; then
-              echo "${DOCKERHUB_IMAGE}:dev"
-            elif [[ "$RELEASE_BRANCH" == main ]]; then
-              echo "${DOCKERHUB_IMAGE}:latest"
-              [[ "$SYNC_DEV" == true ]] && echo "${DOCKERHUB_IMAGE}:dev"
-            fi
-            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
-            echo 'EOF'
-            echo 'ghcr_tags<<EOF'
-            if [[ "$RELEASE_BRANCH" == dev ]]; then
-              echo "${GHCR_IMAGE}:dev"
-            elif [[ "$RELEASE_BRANCH" == main ]]; then
-              echo "${GHCR_IMAGE}:latest"
-              [[ "$SYNC_DEV" == true ]] && echo "${GHCR_IMAGE}:dev"
-            fi
-            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
-            echo 'EOF'
-          } >> "$GITHUB_OUTPUT"
-
-      - name: Create Docker Hub manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          TAG_ARGS=()
-          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
-          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Create GHCR manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          TAG_ARGS=()
-          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
-          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Verify published Docker images run
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION='${{ needs.python-release.outputs.version }}'
-          SHORT_SHA="${RELEASE_SHA::12}"
-          for image in "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"; do
-            "$DOCKER_BINARY" buildx imagetools inspect "$image" >/dev/null
-            INSTALLED_VERSION="$("$DOCKER_BINARY" run --rm "$image" --version)"
-            test "$INSTALLED_VERSION" = "$VERSION"
-          done
-
-      - name: Record published image source
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          # workflow_run can report the default branch's head even when this
-          # run published a different branch/SHA. Preserve the verified target.
-          [[ "$RELEASE_BRANCH" == dev || "$RELEASE_BRANCH" == main ]]
-          [[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]
-          mkdir -p "$RUNNER_TEMP/published-image-source"
-          # shellcheck disable=SC2016
-          "$JQ_BINARY" -n --arg branch "$RELEASE_BRANCH" --arg sha "$RELEASE_SHA" \
-            '{branch: $branch, sha: $sha}' > "$RUNNER_TEMP/published-image-source/source.json"
-
-      - name: Upload published image source
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: published-image-source
-          path: ${{ runner.temp }}/published-image-source/source.json
-          if-no-files-found: error
-
-      - name: Update Docker Hub README
-        uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: archivebox/archivebox
-
-  cascade:
-    name: Cascade verified release
-    if: needs.python-release.outputs.release_ready == 'true'
-    needs:
-      - candidate
-      - python-release
-      - docker-release
-    permissions:
-      actions: read
-      contents: read
-    uses: ArchiveBox/monorepo/.github/workflows/cascade-release.yml@main
-    with:
-      upstream_package: archivebox
-      upstream_version: ${{ needs.python-release.outputs.version }}
-      upstream_artifact_name: python-distributions
-      upstream_run_id: ${{ needs.candidate.outputs.artifact_run_id }}
-    secrets:
-      release_token: ${{ secrets.RELEASE_GH_TOKEN }}

.github/workflows/screenshots.yml

@@ -1,81 +0,0 @@
-name: Capture publicsite screenshots
-
-on:
-  schedule:
-    - cron: '37 5,17 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-jobs:
-  changes:
-    runs-on: ubuntu-24.04
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      key: ${{ steps.check.outputs.key }}
-    steps:
-      - id: check
-        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
-
-  deploy:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Setup Python and uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: false
-
-      - name: Install ArchiveBox and screenshot dependencies
-        run: uv sync --no-cache --frozen
-
-      - name: Build UI screenshot gallery
-        run: bin/collect_ui_screenshots.sh
-        env:
-          BASE_URL: http://archivebox.localhost:5797
-
-      - name: Verify gallery was generated from this exact revision
-        run: |
-          uv run --no-cache --no-sync python - <<'PY'
-          import json
-          import os
-          import re
-          import tomllib
-          from pathlib import Path
-
-          provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
-          version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
-          assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
-          assert provenance["version"] == version, provenance
-          assert provenance["capture_count"] >= 3 * 35, provenance
-          assert len(provenance["files"]) == provenance["capture_count"], provenance
-          assert all(not re.match(r"^\d+-", name) for name in provenance["files"]), provenance
-          PY
-
-      - name: Save validated screenshots
-        uses: actions/upload-artifact@v4
-        with:
-          name: site-screenshots
-          path: publicsite/screenshots
-          if-no-files-found: error
-          retention-days: 90
-
-  complete:
-    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
-    needs: [changes, deploy]
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - run: echo 'Scheduled work completed successfully.'

.github/workflows/staging-acceptance.yml

@@ -1,109 +0,0 @@
-name: Staging acceptance
-
-on:
-  workflow_run:
-    workflows: [Release]
-    types: [completed]
-
-permissions: {}
-
-concurrency:
-  group: staging-acceptance
-  cancel-in-progress: false
-
-jobs:
-  published:
-    if: github.event.workflow_run.conclusion == 'success'
-    runs-on: ubuntu-24.04
-    permissions:
-      actions: read
-    outputs:
-      ready: ${{ steps.source.outputs.ready }}
-      sha: ${{ steps.source.outputs.sha }}
-    steps:
-      - id: image
-        env:
-          GH_TOKEN: ${{ github.token }}
-          RELEASE_RUN_ID: ${{ github.event.workflow_run.id }}
-        run: |
-          set -Eeuo pipefail
-          published=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RELEASE_RUN_ID/jobs?per_page=100" --jq 'any(.jobs[]; .name == "Publish tested multiarch image" and .conclusion == "success")')
-          echo "published=$published" >> "$GITHUB_OUTPUT"
-      - name: Download published image source
-        if: steps.image.outputs.published == 'true'
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: published-image-source
-          path: ${{ runner.temp }}/published-image-source
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ github.event.workflow_run.id }}
-      - name: Select published dev image
-        id: source
-        if: steps.image.outputs.published == 'true'
-        run: |
-          set -Eeuo pipefail
-          # The triggering run may report the default branch as its head;
-          # this artifact names the image source verified by the publisher.
-          source="$RUNNER_TEMP/published-image-source/source.json"
-          jq -e 'type == "object" and (keys | sort) == ["branch", "sha"]' "$source" >/dev/null
-          branch=$(jq -er '.branch | select(type == "string")' "$source")
-          sha=$(jq -er '.sha | select(type == "string")' "$source")
-          [[ "$branch" == dev || "$branch" == main ]]
-          [[ "$sha" =~ ^[0-9a-f]{40}$ ]]
-          echo "sha=$sha" >> "$GITHUB_OUTPUT"
-          if [[ "$branch" == dev ]]; then
-            echo 'ready=true' >> "$GITHUB_OUTPUT"
-          else
-            echo 'ready=false' >> "$GITHUB_OUTPUT"
-          fi
-  acceptance:
-    # The privileged key is never exposed to PRs or arbitrary dispatch refs.
-    needs: published
-    if: needs.published.outputs.ready == 'true'
-    runs-on: [self-hosted, Linux, X64, ugnas-linux]
-    environment: staging
-    timeout-minutes: 110
-    permissions:
-      contents: read
-      deployments: write
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [cabbage]
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
-        with:
-          ref: ${{ needs.published.outputs.sha }}
-          fetch-depth: 0
-          persist-credentials: false
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e
-        with:
-          version: '0.11.3'
-      - name: Prepare browser and restricted staging identity
-        env:
-          STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
-          STAGING_KNOWN_HOSTS: ${{ secrets.STAGING_KNOWN_HOSTS }}
-        run: |
-          set -Eeuo pipefail
-          umask 077
-          printf '%s\n' "$STAGING_SSH_KEY" > "$RUNNER_TEMP/staging-key"
-          printf '%s\n' "$STAGING_KNOWN_HOSTS" > "$RUNNER_TEMP/staging-known-hosts"
-          uv run --no-project --with playwright==1.63.0 playwright install chromium
-      - name: Verify actual deployment, full captures and rendered snapshots
-        env:
-          GH_TOKEN: ${{ github.token }}
-          STAGING_TARGET: ${{ matrix.target }}
-          STAGING_SHA: ${{ needs.published.outputs.sha }}
-          STAGING_KEY_FILE: ${{ runner.temp }}/staging-key
-          STAGING_KNOWN_HOSTS_FILE: ${{ runner.temp }}/staging-known-hosts
-        run: uv run --no-project --with playwright==1.63.0 python bin/check-staging-deployment.py
-      - name: Remove staging identity
-        if: always()
-        run: rm -f "$RUNNER_TEMP/staging-key" "$RUNNER_TEMP/staging-known-hosts"
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
-        if: always()
-        with:
-          name: staging-${{ matrix.target }}-${{ needs.published.outputs.sha }}
-          path: staging-evidence/
-          if-no-files-found: error

.github/workflows/test-parallel.yml

@@ -1,261 +0,0 @@
-name: Parallel Tests
-
-on:
-  workflow_dispatch:
-    inputs:
-      test_file:
-        description: Run one discovered test file
-        required: true
-        default: archivebox/tests/test_cli_list.py
-        type: string
-  workflow_call:
-    inputs:
-      test_file:
-        description: Run one discovered test file; empty runs the complete matrix
-        required: false
-        default: ""
-        type: string
-
-env:
-  PYTHONIOENCODING: utf-8
-  PYTHONLEGACYWINDOWSSTDIO: utf-8
-  USE_COLOR: False
-  UV_NO_SOURCES: "1"
-  CI_PYTHON_VERSION: "3.13"
-
-jobs:
-  discover-tests:
-    name: Discover every test
-    # Selection must run on GitHub so an offline NAS cannot block the fallback.
-    runs-on: ubuntu-24.04
-    outputs:
-      tests: ${{ steps.discover.outputs.tests }}
-      runner: ${{ steps.runner.outputs.use-runner || '["ubuntu-24.04"]' }}
-
-    steps:
-      - name: Select available CI capacity
-        id: runner
-        if: vars.UGNAS_CI_ENABLED == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
-        uses: ArchiveBox/monorepo/.github/actions/select-ci-runner@main
-        with:
-          github-token: ${{ secrets.RUNNER_STATUS_TOKEN }}
-
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: ${{ env.CI_PYTHON_VERSION }}
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Resolve discovery tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          test -n "$ABXPKG_SPEC"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PYTHON_BINARY="$pythonLocation/bin/python"
-          test -x "$PYTHON_BINARY"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
-          {
-            echo "PYTHON_BINARY=$ABXPKG_LIB_DIR/env/bin/python"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-          } >> "$GITHUB_ENV"
-          test -L "$ABXPKG_LIB_DIR/env/bin/python"
-          test -x "$ABXPKG_LIB_DIR/env/bin/python"
-          test -L "$ABXPKG_LIB_DIR/env/bin/jq"
-          test -x "$ABXPKG_LIB_DIR/env/bin/jq"
-
-      - name: Discover every test exactly once
-        id: discover
-        env:
-          TEST_FILE: ${{ inputs.test_file }}
-          UGNAS_CI_MAX_JOBS: ${{ vars.UGNAS_CI_MAX_JOBS || 3 }}
-        run: |
-          set -Eeuo pipefail
-          mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
-          test "${#output[@]}" -eq 2
-          if [[ -n "$TEST_FILE" ]]; then
-            output[1]="$("$JQ_BINARY" -ce '[.[] | select(.paths == [env.TEST_FILE])] | select(length == 1)' <<< "${output[1]}")"
-          fi
-          echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
-          echo "${output[0]}"
-          "$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"
-
-  tests:
-    name: ${{ matrix.test.name }}
-    needs: discover-tests
-    # Opt in trusted branches only; PRs always use GitHub-hosted runners.
-    # Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
-    # The hosted discovery job falls back when workers are offline or the
-    # availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
-    runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
-    env:
-      CHROME_HEADLESS: "true"
-
-    strategy:
-      fail-fast: false
-      matrix:
-        test: ${{ fromJson(needs.discover-tests.outputs.tests) }}
-
-    steps:
-      - name: Set job-local runtime paths
-        env:
-          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
-        run: |
-          workers=2
-          cache_suffix=''
-          if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
-            workers="$(nproc)"
-            cache_suffix="-workers-$workers"
-          fi
-          {
-            echo "CI_TEST_WORKERS=$workers"
-            echo "ABXPKG_CACHE_WORKERS_SUFFIX=$cache_suffix"
-            echo "ABXPKG_LIB_DIR=$RUNNER_TEMP/archivebox-lib"
-            echo "DATA_DIR=$RUNNER_TEMP/archivebox-test-data"
-            echo "PERSONAS_DIR=$RUNNER_TEMP/abx-personas"
-            echo "CHROME_USER_DATA_DIR=$RUNNER_TEMP/abx-personas/Default/chrome_profile"
-          } >> "$GITHUB_ENV"
-
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: ${{ env.CI_PYTHON_VERSION }}
-          architecture: x64
-        id: python
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      # Self-hosted tool storage survives jobs. Normal abxpkg validation and
-      # install locks allow reuse across files; cold-install tests use tmp_path.
-      - name: Reuse persistent runner tools
-        if: runner.environment == 'self-hosted'
-        env:
-          PERSISTENT_ABXPKG_LIB: ${{ runner.tool_cache }}/abxpkg/${{ github.repository }}/${{ steps.python.outputs.python-version }}
-        run: |
-          mkdir -p "$PERSISTENT_ABXPKG_LIB"
-          echo "ABXPKG_LIB_DIR=$PERSISTENT_ABXPKG_LIB" >> "$GITHUB_ENV"
-
-      # Restored binaries still pass normal abxpkg resolution/version checks.
-      # Installation and isolation tests explicitly choose fresh temporary libs.
-      - name: Identify runtime dependency cache
-        shell: bash
-        run: |
-          uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
-          import hashlib
-          import json
-          import tomllib
-          from pathlib import Path
-
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
-          print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
-          PYTHON
-
-      - name: Reuse validated runtime dependencies
-        if: runner.environment != 'self-hosted'
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
-        with:
-          path: ${{ runner.temp }}/archivebox-lib
-          key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}${{ env.ABXPKG_CACHE_WORKERS_SUFFIX }}
-          restore-keys: |
-            abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-
-
-      - name: Resolve optional build dependencies through abxpkg
-        if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
-        env:
-          TEST_EXTRA: ${{ matrix.test.extra }}
-          TEST_PATHS: ${{ matrix.test.paths_arg }}
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          test -n "$ABXPKG_SPEC"
-          dependency_sources=()
-          if [[ -n "$TEST_EXTRA" ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          fi
-          if [[ "$TEST_PATHS" == *test_postgres_backend* ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:postgres_binaries")
-          fi
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-            --install \
-            --no-cache \
-            --json \
-            --lib="$ABXPKG_LIB_DIR" \
-            "${dependency_sources[@]}"
-
-      - name: Install test dependencies
-        env:
-          TEST_EXTRA: ${{ matrix.test.extra }}
-        run: |
-          set -Eeuo pipefail
-          extra_args=()
-          if [[ -n "$TEST_EXTRA" ]]; then
-            extra_args=(--extra "$TEST_EXTRA")
-          fi
-          uv sync --no-cache --locked --dev "${extra_args[@]}"
-      - name: Install Playwright Chromium for browser tests
-        if: contains(matrix.test.paths_arg, 'test_proof_previews') || contains(matrix.test.paths_arg, 'test_snapshot_output_stacks') || contains(matrix.test.paths_arg, 'test_ui_admin_snapshot') || contains(matrix.test.paths_arg, 'test_cli_extract')
-        run: uv run --no-cache --no-sync --no-sources playwright install chromium
-
-      - name: Run ${{ matrix.test.name }}
-        env:
-          TEST_PATHS: ${{ matrix.test.paths_arg }}
-          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
-        run: |
-          set -Eeuo pipefail
-          echo "Running pytest with $CI_TEST_WORKERS workers on $CI_RUNNER_ENVIRONMENT"
-          read -r -a test_paths <<< "$TEST_PATHS"
-          for test_path in "${test_paths[@]}"; do
-            test -f "$test_path"
-          done
-          test_status=0
-          for test_path in "${test_paths[@]}"; do
-            test_name="$(basename "$test_path" .py)"
-            test_slug="${test_name//[^A-Za-z0-9_.-]/_}"
-            export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
-            export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
-            export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
-            mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
-            uv run --no-cache --no-sync --no-sources pytest -vs -n "$CI_TEST_WORKERS" "$GITHUB_WORKSPACE/$test_path" --durations=20 \
-              --basetemp="$RUNNER_TEMP/pytest/$test_slug" \
-              --junitxml="$RUNNER_TEMP/test-results/$test_slug.xml" || test_status=1
-          done
-          exit "$test_status"
-      - name: Save test results
-        if: always()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: test-results-${{ strategy.job-index }}
-          path: ${{ runner.temp }}/test-results/*.xml
-          if-no-files-found: error
-          retention-days: 14
-      - name: Save failed runtime diagnostics
-        if: failure()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: runtime-logs-${{ strategy.job-index }}
-          path: ${{ runner.temp }}/pytest/**/*.log
-          if-no-files-found: ignore
-          retention-days: 14

.github/workflows/test.yml

@@ -1,60 +1,158 @@
-name: Install and CLI Compatibility
-
-on:
-  workflow_call:
-    inputs:
-      platform:
-        description: Platform for install and CLI compatibility
-        required: false
-        default: linux
-        type: string
+name: Run tests
+on: [push]
 
 env:
   DOCKER_IMAGE: archivebox-ci
-  ARCHIVEBOX_IMAGE: archivebox-ci
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
-  UV_NO_SOURCES: "1"
 
 jobs:
-  install_cli_platform:
-    name: CLI install / ${{ matrix.os_name }} / Python ${{ matrix.python }}
+  python_tests:
     runs-on: ${{ matrix.os }}
 
     strategy:
-      fail-fast: false
       matrix:
-        include: ${{ fromJSON(inputs.platform == 'macos' && '[{"os":"macos-15","os_name":"macOS","python":"3.13"},{"os":"macos-15","os_name":"macOS","python":"3.14"}]' || '[{"os":"ubuntu-24.04","os_name":"linux","python":"3.14"}]') }}
+        os: [ubuntu-22.04]
+        # os: [ubuntu-22.04, macos-latest, windows-latest]
+        python: [3.11]
 
     steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
-      ### Set up Python
+      ### Setup Python & JS Languages
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
+          architecture: x64
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
         with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          node-version: 20.10.0
 
-      - name: Install dependencies with uv
-        run: uv sync --no-cache --locked --dev
+      - name: Setup PDM
+        uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.11'
+          cache: true
 
-      - name: ArchiveBox full install check
+      ### Install Python & JS Dependencies
+      - name: Get pip cache dir
+        id: pip-cache
         run: |
-          set -Eeuo pipefail
-          DATA_DIR="$PWD/tests/out/data"
-          mkdir -p "$DATA_DIR"
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox init
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox install
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox version
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox status
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: Cache pip
+        uses: actions/cache@v3
+        id: cache-pip
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python }}-venv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
+
+      - name: Install pip dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel pytest bottle build
+          python -m pip install -r requirements.txt
+          python -m pip install -e .[sonic,ldap]
+
+      - name: Get npm cache dir
+        id: npm-cache
+        run: |
+          echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
+
+      - name: Cache npm
+        uses: actions/cache@v3
+        id: cache-npm
+        with:
+          path: ${{ steps.npm-cache.outputs.dir }}
+          key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node_modules
+
+      - name: Install npm requirements
+        run: |
+          npm install
+          echo "SINGLEFILE_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/single-file" >> $GITHUB_ENV
+          echo "READABILITY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor" >> $GITHUB_ENV
+          echo "MERCURY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/mercury-parser" >> $GITHUB_ENV
+
+      ### Run the tests
+      - name: Directory listing for debugging
+        run: |
+          pwd
+          ls
+
+      - name: Archivebox version
+        run: |
+          archivebox version
+
+      - name: Test built package with pytest
+        # TODO: remove this exception for windows once we get tests passing on that platform
+        if: ${{ !contains(matrix.os, 'windows') }}
+        run: |
+          python -m pytest -s --basetemp=tests/out --ignore=archivebox/vendor --ignore=deb_dist --ignore=pip_dist --ignore=brew_dist
+
+  docker_tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
+      # - uses: satackey/action-docker-layer-caching@v0.0.8
+
+      - name: Build image
+        run: |
+          docker build . -t "$DOCKER_IMAGE"
+
+      - name: Init data dir
+        run: |
+          mkdir "${{ github.workspace }}/data"
+          docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init
+
+      - name: Run test server
+        run: |
+          sudo bash -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
+          docker run --name www-nginx -p 80:80 -d nginx
+
+      - name: Add link
+        run: |
+          docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local
+
+      - name: Add stdin link
+        run: |
+          echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add
+
+      - name: List links
+        run: |
+          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
+          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }
+
+      - name: Start docker-compose stack
+        run: |
+          docker-compose run archivebox init
+          docker-compose up -d
+          sleep 5
+          curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
+          curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'
+
+      - name: Check added urls show up in index
+        run: |
+          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
+          curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
+          docker-compose down || true

.gitignore

@@ -1,36 +1,22 @@
 .DS_Store
 ._*
-/.heartbeat.json
-/package.json
-/package-lock.json
 *.pyc
 __pycache__/
 .mypy_cache/
 .eggs/
 tests/out/
-/docs/screenshots/
-/publicsite/screenshots/
-
-# Coverage
-.coverage
-.coverage.*
-coverage.json
-coverage/
-htmlcov/
 
 # Python and Node dependencies
 venv/
 .venv/
 .docker-venv/
 node_modules/
-typings/
 
 # Ignore dev lockfiles (should always be built fresh)
-pdm.dev.lock
 requirements-dev.txt
+pdm.dev.lock
 
 # Packaging artifacts
-requirements.txt
 .pdm-python
 .pdm-build
 archivebox.egg-info
@@ -39,34 +25,10 @@ build/
 dist/
 
 # Data folders
-lib/
-out/
-tmp/
 data/
 data*/
-archivebox/tests/data/
-archive/
 output/
-logs/
-/-
-/personas/
-/sources/
-index.sqlite3
-queue.sqlite3
-*.sqlite*
-data.*
-.archivebox_id
-ArchiveBox.conf
-*.stdout
-*.stderr
-*.log
-.tmp/
 
 # vim
 *.sw?
-.vscode
-
-# Local Claude Code task locks (not project state)
-.claude/scheduled_tasks.lock
-
-/_site/
+.vscode/

.gitmodules

@@ -0,0 +1,28 @@
+[submodule "docs"]
+    path = docs
+    url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
+
+[submodule "deb_dist"]
+    path = deb_dist
+    url = https://github.com/ArchiveBox/debian-archivebox.git
+[submodule "brew_dist"]
+	path = brew_dist
+	url = https://github.com/ArchiveBox/homebrew-archivebox.git
+[submodule "pip_dist"]
+	path = pip_dist
+	url = https://github.com/ArchiveBox/pip-archivebox.git
+[submodule "docker"]
+	path = docker
+	url = https://github.com/ArchiveBox/docker-archivebox.git
+[submodule "archivebox/vendor/base32-crockford"]
+	path = archivebox/vendor/base32-crockford
+	url = https://github.com/jbittel/base32-crockford
+[submodule "archivebox/vendor/pocket"]
+	path = archivebox/vendor/pocket
+	url = https://github.com/tapanpandita/pocket
+[submodule "archivebox/vendor/django-taggit"]
+	path = archivebox/vendor/django-taggit
+	url = https://github.com/jazzband/django-taggit
+[submodule "archivebox/vendor/python-atomicwrites"]
+	path = archivebox/vendor/python-atomicwrites
+	url = https://github.com/untitaker/python-atomicwrites

.npmignore

@@ -0,0 +1,19 @@
+tests/
+archivebox/
+archivebox.egg-info/
+build/
+dist/
+docs/
+etc/
+.github
+.gitmodules
+.dockerignore
+.flake8
+CNAME
+_config.yml
+docker-compose.yaml
+docker-compose.yml
+Dockerfile
+MANIFEST.in
+Pipfile
+setup.py

.pre-commit-config.yaml

@@ -1,69 +0,0 @@
-default_language_version:
-  python: python3.13
-
-repos:
-  - repo: https://github.com/asottile/yesqa
-    rev: v1.5.0
-    hooks:
-      - id: yesqa
-
-  - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.1
-    hooks:
-      - id: codespell
-        additional_dependencies:
-          - tomli
-
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.20.0
-    hooks:
-      - id: pyupgrade
-        args: [--py313-plus]
-
-  - repo: https://github.com/asottile/add-trailing-comma
-    rev: v3.1.0
-    hooks:
-      - id: add-trailing-comma
-
-  - repo: local
-    hooks:
-      - id: ruff-format
-        name: ruff-format
-        entry: uv run --active --no-sync --no-sources ruff format
-        language: system
-        types_or: [python, pyi]
-      - id: ruff-check
-        name: ruff-check
-        entry: uv run --active --no-sync --no-sources ruff check --fix
-        language: system
-        types_or: [python, pyi]
-
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
-    hooks:
-      - id: check-ast
-      - id: check-toml
-      - id: check-yaml
-        exclude: ^\.github/workflows/homebrew\.yml$
-      - id: check-json
-      - id: check-merge-conflict
-      - id: check-symlinks
-      - id: destroyed-symlinks
-      - id: check-case-conflict
-      - id: check-illegal-windows-names
-      - id: check-shebang-scripts-are-executable
-        exclude: ^(archivebox/.*\.py|archivebox/tests/.*\.py|archivebox/personas/export_browser_state\.js)$
-      - id: mixed-line-ending
-      - id: fix-byte-order-marker
-      - id: end-of-file-fixer
-        # Preserve the exact bytes of captured cryptographic proof fixtures.
-        exclude: ^archivebox/tests/fixtures/legacy_proofs/.*\.json$
-      - id: detect-private-key
-      - id: debug-statements
-      - id: forbid-submodules
-        exclude: ^docs$
-      - id: check-added-large-files
-        args: ["--maxkb=600"]
-      - id: name-tests-test
-        args: ["--pytest-test-first"]
-        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)

.readthedocs.yaml

@@ -0,0 +1,33 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+submodules:
+  include: all
+  recursive: true
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    nodejs: "20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+
+# Optionally build your docs in additional formats such as PDF and ePub
+formats:
+  - pdf
+  - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt

AGENTS.md

@@ -1,83 +0,0 @@
-# ArchiveBox Agent Guide
-
-ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
-
-## Shared Standards
-
-- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
-- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
-- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
-- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
-- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
-- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
-- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
-- Start behavior fixes with a red failing test when a test is requested or practical.
-- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
-- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
-
-## Concurrency Contract
-
-- A collection has one orchestrator at a time. Local PID/process checks may warn about obvious same-machine duplicates, but must not claim to enforce ownership across machines or shared filesystems.
-- SQLite remains supported with concurrent short writes from CLI and server processes. Keep the SQLite database on a local filesystem, never NFS/SMB, and use short autocommit/CAS updates instead of long transactions or database locks.
-- Network calls, hook execution, filesystem migrations, and other long work belong in the orchestrator. Never hold a database transaction or lock across that work.
-- PostgreSQL and shared data directories are the path to future multi-machine scheduling. Coordinate that work through database-backed per-crawl/per-snapshot claims; do not introduce file leases or timer-based orchestrator election.
-
-## Development Setup
-
-```bash
-uv sync --dev --all-extras
-mkdir -p data
-cd data
-uv run --project .. archivebox init --install
-```
-
-Run collection commands from inside an initialized data directory:
-
-```bash
-cd data
-uv run --project .. archivebox status
-uv run --project .. archivebox add --plugins=parse_txt_urls 'https://example.com/'
-uv run --project .. archivebox run
-```
-
-## User-Facing Setup
-
-Recommended CLI install:
-
-```bash
-uv tool install --python 3.13 --prerelease explicit --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
-mkdir -p ~/archivebox/data
-cd ~/archivebox/data
-archivebox init --install
-archivebox add --plugins=parse_txt_urls 'https://example.com/'
-```
-
-Alternative install methods:
-
-- Docker Compose / Docker
-- Homebrew
-- Debian package
-- pip
-
-## Basic Usage
-
-```bash
-cd ~/archivebox/data
-archivebox version
-archivebox help
-archivebox status
-archivebox install
-archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
-archivebox list --json --with-headers
-archivebox search 'example'
-archivebox update --filter-type=domain example.com
-archivebox remove --yes --delete --filter-type=exact 'https://example.com/docs-basic-usage'
-archivebox run
-```
-
-## Verification
-
-Run the test suite with `uv run pytest archivebox/tests -x` and repository checks
-with `uv run prek run --all-files`. For focused work, select the relevant tests.
-
-Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.

Dockerfile

@@ -1,214 +1,22 @@
-# syntax=docker/dockerfile:1.7
+# This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
+#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, yt-dlp, single-file
+# Usage:
+#     git submodule update --init --recursive
+#     git pull --recurse-submodules
+#     docker build . -t archivebox --no-cache
+#     docker run -v "$PWD/data":/data archivebox init
+#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
+#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
+#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
+# Multi-arch build:
+#     docker buildx create --use
+#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:latest -t archivebox/archivebox:dev
+#
+# Read more about [developing Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
 
-# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# WHY: abx-dl owns Python, Node, Chromium, and extraction dependencies so the
-# standalone downloader stays independent of ArchiveBox's server features.
-# ArchiveBox-only dependencies (Sonic, OpenCode, supervisor, Django) belong in
-# the layers added by this Dockerfile, never in the shared abx-dl base image.
-# REQUIRED INSTALLATION CONTRACT — do not work around failures:
-# 1. IMAGE CONSTRUCTION: preinstall ALL plugins supported by runtime="archivebox"
-#    and ALL their required_binaries, regardless of default enabled state.
-#    Inherit the complete downloader dependencies from the tested abx-dl image;
-#    install every remaining ArchiveBox-only dependency in this image's layers,
-#    including OpenCode and Sonic. Never put server dependencies in abx-dl.
-#    Use the normal plugin dependency pipeline: install command -> config.json
-#    required_binaries -> abxpkg. Never add manual apt/pip/npm installs to hide
-#    failures in that pipeline. OS/interpreter bootstrap is a separate concern,
-#    not a second hand-maintained list of plugin dependencies.
-# 2. FINISHED IMAGE: archivebox install and the full plugin installation check
-#    must discover everything already present and be NO-OPS offline as UID 911.
-#    No downloads, package installs, sudo, or root repair may be needed to enable
-#    ANY plugin or start a new crawl. Never run verification as root or manually
-#    preinstall packages before it. If the inherited image lacks a downloader
-#    dependency, fix/release abx-dl and consume that tested image; do not patch
-#    over the missing base dependency here or repair deployed containers by SSH.
-# 3. NON-CONTAINER CI: exercise real runtime auto-installation through the same
-#    CLI/required_binaries/abxpkg path users invoke. GitHub's runner has
-#    passwordless sudo; abxpkg must escalate automatically when needed. Never
-#    manually preinstall dependencies or wrap the tested command in sudo to make
-#    a failure disappear. This is distinct from the finished-image no-op test.
-# 4. SIZE: the complete compressed image budget is 1025 MiB. Preserve ALL
-#    user-facing functionality. Only mechanical/internal
-#    optimizations are allowed. Never remove dependencies, disable features or
-#    plugins, weaken assertions, or skip checks to meet a size limit. If the
-#    complete image exceeds the configured limit, LET CI FAIL and explicitly
-#    ask the maintainer what to do. Do not raise the limit unilaterally.
-# Build abx-dl first, then point this file at it:
-#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
-#       --build-context abxbus=../abxbus \
-#       --build-context abxpkg=../abxpkg \
-#       --build-context abx-plugins=../abx-plugins \
-#       -t archivebox/abx-dl:dev
-#   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:dev \
-#       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:main
-
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
-
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-ARG ARCHIVEBOX_COMMIT_HASH=""
-
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_NO_COMPILE=1 \
-    PIP_ONLY_BINARY=aiohttp \
-    npm_config_loglevel=error
-
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=24
-
-ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_ARCHIVEBOX_UID=911 \
-    DEFAULT_ARCHIVEBOX_GID=911 \
-    IN_DOCKER=True
-
-ENV CODE_DIR=/app \
-    ABX_RUNTIME=archivebox \
-    DATA_DIR=/data \
-    CONFIG_DIR=/opt/archivebox \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
-    PERSONAS_DIR=/data/personas \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/venv/bin/python3 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-ENV HOME=/home/archivebox \
-    XDG_CONFIG_HOME=/home/archivebox/.config \
-    XDG_CACHE_HOME=/opt/archivebox/lib/cache \
-    TIMEOUT=600
-
-ENV UV_COMPILE_BYTECODE=false \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv \
-    VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:$PATH"
-
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
-WORKDIR "$CODE_DIR"
-
-RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
-    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env node \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env uv \
-    ) | tee -a /VERSION.txt
-
-FROM archivebox-runtime-base AS archivebox-builder
-
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
-    <<'EOF'
-echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
-export UV_NO_CACHE=true
-rm -f /etc/apt/apt.conf.d/docker-clean
-apt-get update -qq
-apt-get install -qq -y --no-install-recommends x11-utils
-# Extend abx-dl's existing venv instead of clearing it. Clearing and later
-# copying a complete replacement over the base duplicates every venv byte in
-# the final overlay history even when most packages are unchanged.
-# Do not rewrite inherited native libraries merely to strip them again.
-/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries
-
-# The old Docker-only dependency rewrite predated this checkout's lockfile:
-# it fetched version JSON to avoid stale PyPI Simple listings. That extra API
-# can itself lag or return 503s. The verified lock already records exact public
-# artifact URLs and hashes, including extras and platform markers. Consume it
-# directly, retaining the freshness check instead of resolving a second graph.
-/usr/bin/uv sync \
-    --project /app \
-    --locked \
-    --no-cache \
-    --no-dev \
-    --inexact \
-    --no-install-project \
-    --no-install-workspace \
-    --no-install-package abxbus \
-    --no-install-package abxpkg \
-    --no-install-package abx-plugins \
-    --no-install-package abx-dl \
-    --no-sources
-builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
-ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
-/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
-while IFS= read -r -d '' native_library; do
-    if grep -Fxq "$native_library" /tmp/archivebox-inherited-libraries; then
-        continue
-    fi
-    magic=''
-    if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
-        "$builder_abxpkg_lib_dir/env/bin/strip" --strip-unneeded "$native_library" || exit $?
-    fi
-done < /tmp/archivebox-native-libraries
-rm -f /tmp/archivebox-native-libraries /tmp/archivebox-inherited-libraries
-rm -f /venv/bin/uv /venv/bin/uvx
-abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils
-abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get autoremove -y
-rm -rf "$builder_abxpkg_lib_dir"
-rm -rf /venv/lib/python3.*/site-packages/pip* \
-    /venv/lib/python3.*/site-packages/wheel* \
-    /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
-rm -rf /var/lib/apt/lists/*
-EOF
-
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-# Compile only ArchiveBox's installed package. Touching all of /venv here would
-# copy every inherited abx-dl file into a new layer merely to change its mtime.
-RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && COMMIT_HASH="$( \
-        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-            echo "$ARCHIVEBOX_COMMIT_HASH"; \
-        elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
-            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
-            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-                echo "$HEAD_REF"; \
-            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
-                REF_PATH="${HEAD_REF#ref: }"; \
-                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
-            fi; \
-        fi)" \
-    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
-    && /usr/bin/uv pip install --no-cache --no-deps "$CODE_DIR" \
-    && rm -f /venv/bin/uv /venv/bin/uvx \
-    && cd / \
-    && ARCHIVEBOX_PY_DIR="$(/venv/bin/python -c 'import pathlib, archivebox; print(pathlib.Path(archivebox.__file__).parent)')" \
-    && /venv/bin/python -m compileall --invalidation-mode checked-hash -q "$ARCHIVEBOX_PY_DIR" \
-    && test -f "$(/venv/bin/python -c 'import archivebox; print(archivebox.__cached__)')" \
-    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
-
-# The builder installs and purges stripping tools in one layer, so its final
-# filesystem is already runtime-clean. Preserve that ancestry: starting again
-# from archivebox-runtime-base and COPYing /venv would bake a second full venv
-# over the inherited one instead of recording only ArchiveBox's package delta.
-FROM archivebox-builder
+FROM python:3.11-slim-bookworm
+# Uses Debian 12 w/ faster-updating apt-lists added below: https://packages.debian.org/bookworm-backports/
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -219,92 +27,296 @@ LABEL name="archivebox" \
     org.opencontainers.image.vendor="ArchiveBox" \
     org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
     org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile"
+    com.docker.image.source.entrypoint="Dockerfile" \
+    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
+    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
+    com.docker.desktop.extension.api.version=">= 1.4.7" \
+    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
+    com.docker.extension.publisher-url="https://archivebox.io" \
+    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
+    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
+    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
+    com.docker.extension.categories='database,utility-tools'
+    
+ARG TARGETPLATFORM
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
 
-COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
+######### Environment Variables #################################
 
-# The builder invokes abxpkg as root for temporary ELF stripping, which can
-# rewrite derived state ownership. Restore UID 911 before resolving binaries
-# as the runtime user; the old copy-based final stage hid this dependency.
-RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777,id=archivebox-runtime-config-$TARGETARCH \
-    echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
-    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
-    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
-    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
-    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
-    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
-    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
-    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
-    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
-    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
-    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
-    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
-    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
-    && find "$ABXPKG_LIB_DIR" \( ! -user "$DEFAULT_ARCHIVEBOX_UID" -o ! -group "$DEFAULT_ARCHIVEBOX_GID" \) -exec chown -h "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" {} + \
-    && UV_NO_CACHE=true setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
-    && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+# Global system-level config
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    npm_config_loglevel=error
 
-# Install the complete ArchiveBox catalog, including disabled server plugins.
-# Explicit plugin names make installation independent of enabled defaults.
-RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-opencode-$TARGETARCH \
-    chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" /var/tmp/abxpkg-cache \
-    && chmod 1777 /var/tmp/abxpkg-cache \
-    && export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
-    && env ABXPKG_TMP_CACHE_DIR=/var/tmp/abxpkg-cache XDG_CACHE_HOME=/var/tmp/abxpkg-cache \
-        setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
-    # pnpm includes musl variants that Debian's glibc runtime cannot use.
-    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules" -type l -name 'opencode-linux-*-musl' -delete \
-    && find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.pnpm" -maxdepth 1 -type d -name 'opencode-linux-*-musl@*' -exec rm -rf {} + \
-    # pnpm's explicit store lives under ABXPKG_LIB_DIR, regardless of XDG_CACHE_HOME.
-    # Retaining its download copies made image size depend on whether the build
-    # preserved hardlinks to installed files. Remove the store in this layer;
-    # installed files retain their contents, and the offline check below proves
-    # the complete catalog still works without the disposable package payloads.
-    && rm -rf "$ABXPKG_LIB_DIR/cache/pnpm"
-RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
-    bash -c '"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version && abx-dl install $ABX_DOCKER_PLUGINS'
+# Version config
+ENV PYTHON_VERSION=3.11 \
+    NODE_VERSION=20
 
-WORKDIR "$DATA_DIR"
-RUN echo "[+] Initializing image collection..." \
-    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && HOME="$TMP_DIR" archivebox init \
-    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
-        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
-        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
-        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
-        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
-        "$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
-        2>/dev/null || true) \
-    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+# User config
+ENV ARCHIVEBOX_USER="archivebox" \
+    DEFAULT_PUID=911 \
+    DEFAULT_PGID=911
 
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+# Global paths
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    GLOBAL_VENV=/venv \
+    PLAYWRIGHT_BROWSERS_PATH=/browsers
 
-RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
-    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+# Application-level paths
+ENV APP_VENV=/app/.venv \
+    NODE_MODULES=/app/node_modules
 
-RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
+# Build shell config
+ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
+
+######### System Environment ####################################
+
+# Detect ArchiveBox version number by reading package.json
+COPY --chown=root:root --chmod=755 package.json "$CODE_DIR/"
+RUN grep '"version": ' "${CODE_DIR}/package.json" | awk -F'"' '{print $4}' > /VERSION.txt
+
+# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
+RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
+    && rm -f /etc/apt/apt.conf.d/docker-clean
+
+# Print debug info about build and save it to disk, for human eyes only, not used by anything else
+RUN (echo "[i] Docker build for ArchiveBox $(cat /VERSION.txt) starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && echo \
+    && echo "GLOBAL_VENV=${GLOBAL_VENV} APP_VENV=${APP_VENV} NODE_MODULES=${NODE_MODULES}" \
+    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
+    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
+    && echo \
+    && uname -a \
+    && cat /etc/os-release | head -n7 \
+    && which bash && bash --version | head -n1 \
+    && which dpkg && dpkg --version | head -n1 \
+    && echo -e '\n\n' && env && echo -e '\n\n' \
     ) | tee -a /VERSION.txt
 
+# Create non-privileged user for archivebox and chrome
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
+    && groupadd --system $ARCHIVEBOX_USER \
+    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
+    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
+    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
+    | tee -a /VERSION.txt
+    # DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
+    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
+
+# Install system apt dependencies (adding backports to access more recent apt updates)
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
+    && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \
+    && mkdir -p /etc/apt/keyrings \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        # 1. packaging dependencies
+        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
+        # 2. docker and init system dependencies
+        zlib1g-dev dumb-init gosu cron unzip grep ncat \
+        # 3. frivolous CLI helpers to make debugging failed archiving easier
+        # nano iputils-ping dnsutils htop procps jq yq \
+    && rm -rf /var/lib/apt/lists/*
+
+######### Language Environments ####################################
+
+# Install Node environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
+    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        nodejs libatomic1 python3-minimal \
+    && rm -rf /var/lib/apt/lists/* \
+    # Update NPM to latest version
+    && npm i -g npm --cache /root/.npm \
+    # Save version info
+    && ( \
+        which node && node --version \
+        && which npm && npm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install Python environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Setting up Python $PYTHON_VERSION runtime..." \
+    # tell PDM to allow using global system python site packages
+    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
+    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
+    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
+    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
+    # && python3 -m venv $GLOBAL_VENV \
+    # install global dependencies / python build dependencies in GLOBAL_VENV
+    # && pip install --upgrade pip setuptools wheel \
+    # Save version info
+    && ( \
+        which python3 && python3 --version | grep " $PYTHON_VERSION" \
+        && which pip && pip --version \
+        # && which pdm && pdm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+######### Extractor Dependencies ##################################
+
+# Install apt dependencies
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing APT extractor dependencies globally using apt..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        curl wget git yt-dlp ffmpeg ripgrep \
+        # Packages we have also needed in the past:
+        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
+    && rm -rf /var/lib/apt/lists/* \
+    # Save version info
+    && ( \
+        which curl && curl --version | head -n1 \
+        && which wget && wget --version 2>&1 | head -n1 \
+        && which yt-dlp && yt-dlp --version 2>&1 | head -n1 \
+        && which git && git --version 2>&1 | head -n1 \
+        && which rg && rg --version 2>&1 | head -n1 \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install chromium browser using playwright
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
+        # libxss1 dbus dbus-x11 upower \
+    # && service dbus start \
+    && if [[ "$TARGETPLATFORM" == *amd64* || "$TARGETPLATFORM" == *arm64* ]]; then \
+        # install Chromium using playwright
+        pip install playwright \
+        && cp -r /root/.cache/ms-playwright "$PLAYWRIGHT_BROWSERS_PATH" \
+        && playwright install --with-deps chromium \
+        && export CHROME_BINARY="$(python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')"; \
+    else \
+        # fall back to installing Chromium via apt-get on platforms not supported by playwright (e.g. risc, ARMv7, etc.)
+        # apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        #     chromium \
+        # && export CHROME_BINARY="$(which chromium)"; \
+        echo 'armv7 no longer supported in versions after v0.7.3' \
+        exit 1; \
+    fi \
+    && rm -rf /var/lib/apt/lists/* \
+    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
+    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
+    && chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config" \
+    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
+    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
+    # Save version info
+    && ( \
+        which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install Node dependencies
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR"/
+RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
+    && npm ci --prefer-offline --no-audit --cache /root/.npm \
+    && ( \
+        which node && node --version \
+        && which npm && npm version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+######### Build Dependencies ####################################
+
+# Install ArchiveBox Python dependencies
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR"/
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt for ${TARGETPLATFORM}..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+        build-essential \
+        libssl-dev libldap2-dev libsasl2-dev \
+        python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
+    # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
+    # && pdm use --venv in-project \
+    # && pdm run python -m ensurepip \
+    # && pdm sync --fail-fast --no-editable --group :all --no-self \
+    # && pdm export -o requirements.txt --without-hashes \
+    # && source $GLOBAL_VENV/bin/activate \
+    && pip install -r requirements.txt \
+    && apt-get purge -y \
+        build-essential \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install ArchiveBox Python package from source
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
+    # && apt-get update -qq \
+    # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
+    # && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
+    #     build-essential  \
+    # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
+    && pip install -e "$CODE_DIR"[sonic,ldap] \
+    # save docker image size and always remove compilers / build tools after building is complete
+    # && apt-get purge -y build-essential \
+    # && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+####################################################
+
+# Setup ArchiveBox runtime config
+WORKDIR "$DATA_DIR"
+ENV IN_DOCKER=True \
+    DISPLAY=novnc:0.0 \
+    CUSTOM_TEMPLATES_DIR=/data/templates \
+    CHROME_USER_DATA_DIR=/data/personas/Default/chromium \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no \
+    ALLOWED_HOSTS=*
+    ## No need to set explicitly, these values will be autodetected by archivebox in docker:
+    # WGET_BINARY="wget" \
+    # YOUTUBEDL_BINARY="yt-dlp" \
+    # CHROME_BINARY="/usr/bin/chromium-browser" \
+    # USE_SINGLEFILE=True \
+    # SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
+    # USE_READABILITY=True \
+    # READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
+    # USE_MERCURY=True \
+    # MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"
+
+# Print version for nice docker finish summary
+RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
+    ) | tee -a /VERSION.txt
+RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
+
+####################################################
+
+# Open up the interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
-EXPOSE 5797
+EXPOSE 8000
 
 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:5797/health/' | grep -q 'OK'
+    CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'
 
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:5797"]
+CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2026 Nick Sweeting
+Copyright (c) 2020 Nick Sweeting
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

SECURITY.md

@@ -0,0 +1,34 @@
+# Security Policy
+
+---
+
+## Security Information
+
+Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
+
+https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+
+Also see this section of the README about important caveats when running ArchiveBox:
+
+https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
+
+You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
+
+- https://github.com/ArchiveBox/ArchiveBox#archive-layout
+- https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
+
+---
+
+## Reporting a Vulnerability
+
+We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.
+
+1. Go to the Security tab on our Github repo: https://github.com/ArchiveBox/ArchiveBox/security
+
+2. Click the ["Report a Vulnerability"](https://github.com/ArchiveBox/ArchiveBox/security/advisories/new) button
+
+3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
+
+You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).

archivebox/.flake8

@@ -3,4 +3,4 @@ ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E
 select = F,E9,W
 max-line-length = 130
 max-complexity = 10
-exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv,data,data*
+exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv

archivebox/LICENSE

@@ -0,0 +1 @@
+../LICENSE

archivebox/__init__.py

@@ -1,150 +1 @@
-#!/usr/bin/env python3
-
-# Welcome to the ArchiveBox source code! Thanks for checking it out!
-#
-# "We are swimming upstream against a great torrent of disorganization.
-# In this, our main obligation is to establish arbitrary enclaves of order and system.
-# It is the greatest possible victory to be, to continue to be, and to have been.
-# No defeat can deprive us of the success of having existed for some moment of time
-# in a universe that seems indifferent to us."
-# --Norber Weiner
-
-__package__ = "archivebox"
-
-import os
-import sys
-from pathlib import Path
-from typing import Protocol, cast
-
-
-class _ReconfigurableStream(Protocol):
-    def reconfigure(self, *, line_buffering: bool) -> object: ...
-
-
-# Force unbuffered output for real-time logs
-if hasattr(sys.stdout, "reconfigure"):
-    cast(_ReconfigurableStream, sys.stdout).reconfigure(line_buffering=True)
-    cast(_ReconfigurableStream, sys.stderr).reconfigure(line_buffering=True)
-os.environ["PYTHONUNBUFFERED"] = "1"
-
-ASCII_LOGO = """
- █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
-██╔══██╗██╔══██╗██╔════╝██║  ██║██║██║   ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
-███████║██████╔╝██║     ███████║██║██║   ██║█████╗   ██████╔╝██║   ██║ ╚███╔╝ 
-██╔══██║██╔══██╗██║     ██╔══██║██║╚██╗ ██╔╝██╔══╝   ██╔══██╗██║   ██║ ██╔██╗ 
-██║  ██║██║  ██║╚██████╗██║  ██║██║ ╚████╔╝ ███████╗ ██████╔╝╚██████╔╝██╔╝ ██╗
-╚═╝  ╚═╝╚═╝  ╚═╝ ╚═════╝╚═╝  ╚═╝╚═╝  ╚═══╝  ╚══════╝ ╚═════╝  ╚═════╝ ╚═╝  ╚═╝
-"""
-
-PACKAGE_DIR = Path(__file__).resolve().parent
-
-os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
-os.environ["TZ"] = "UTC"
-
-# detect ArchiveBox user's UID/GID based on data dir ownership
-from .config.permissions import drop_privileges  # noqa
-
-drop_privileges()
-
-from .misc.checks import check_not_root, check_io_encoding  # noqa
-
-check_not_root()
-check_io_encoding()
-
-from .config.version import VERSION  # noqa
-
-
-__version__ = VERSION
-__author__ = "ArchiveBox"
-__license__ = "MIT"
-
-
-def __getattr__(name: str):
-    if name == "CONSTANTS":
-        from .config.constants import CONSTANTS
-
-        os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
-        return CONSTANTS
-    if name == "DATA_DIR":
-        from .config.paths import DATA_DIR
-
-        return DATA_DIR
-    if name == "VERSION":
-        return VERSION
-    if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
-        from abx_plugins import get_plugins_dir
-        from .config.constants import CONSTANTS
-
-        builtin_plugins_dir = Path(get_plugins_dir()).resolve()
-        user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
-        plugins = {
-            "builtin": builtin_plugins_dir,
-            "user": user_plugins_dir,
-        }
-        values = {
-            "BUILTIN_PLUGINS_DIR": builtin_plugins_dir,
-            "USER_PLUGINS_DIR": user_plugins_dir,
-            "ALL_PLUGINS": plugins,
-            "LOADED_PLUGINS": plugins,
-        }
-        return values[name]
-    raise AttributeError(name)
-
-
-__all__ = (
-    "ASCII_LOGO",
-    "ASCII_ICON",
-    "PACKAGE_DIR",
-    "DATA_DIR",
-    "CONSTANTS",
-    "VERSION",
-    "BUILTIN_PLUGINS_DIR",
-    "USER_PLUGINS_DIR",
-    "ALL_PLUGINS",
-    "LOADED_PLUGINS",
-)
-
-ASCII_ICON = """
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-██████████████████████████████████████████████████████████████████████████████████████████████████ 
-         ██                                                                            ██          
-         ██                                                                            ██        
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                    ████████████████████████████████████                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ██       █████████████████████████ █                    ██          
-         ██                    ████████████████████████████████████                    ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                 ██████████████████████████████████████████                 ██          
-         ██                 ██████████████████████████████████████████                 ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██          
-         ██                                                                            ██        
-         ████████████████████████████████████████████████████████████████████████████████          
-"""
+__package__ = 'archivebox'

archivebox/__main__.py

@@ -1,20 +1,11 @@
 #!/usr/bin/env python3
-"""This is the entrypoint for python -m archivebox ..."""
 
-__package__ = "archivebox"
+__package__ = 'archivebox'
 
-import archivebox  # noqa # make sure monkey patches are applied before anything else
 import sys
 
 from .cli import main
 
-ASCII_LOGO_MINI = r"""
-     _             _     _           ____
-    / \   _ __ ___| |__ (_)_   _____| __ )  _____  __
-   / _ \ | '__/ __| '_ \| \ \ / / _ \  _ \ / _ \ \/ /
-  / ___ \| | | (__| | | | |\ V /  __/ |_) | (_) >  <
- /_/   \_\_|  \___|_| |_|_| \_/ \___|____/ \___/_/\_\
-"""
 
-if __name__ == "__main__":
-    main(args=sys.argv[1:])
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/api/__init__.py

@@ -1 +0,0 @@
-__package__ = "archivebox.api"

archivebox/api/admin.py

@@ -1,120 +0,0 @@
-__package__ = "archivebox.api"
-
-from django import forms
-from django.contrib import admin
-from django.http import HttpRequest
-from django.utils.text import capfirst
-from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
-from signal_webhooks.settings import webhook_settings
-from signal_webhooks.utils import get_webhook_model, model_from_reference
-
-from archivebox.base_models.admin import BaseModelAdmin
-
-from archivebox.api.models import APIToken
-
-
-def _webhook_fields(*names: str) -> tuple[str, ...]:
-    model_fields = {field.name for field in get_webhook_model()._meta.fields}
-    return tuple(name for name in names if name in model_fields)
-
-
-class APITokenAdmin(BaseModelAdmin):
-    list_display = ("created_at", "id", "created_by", "token_redacted", "expires")
-    sort_fields = ("id", "created_at", "created_by", "expires")
-    readonly_fields = ("created_at", "modified_at")
-    search_fields = ("id", "created_by__username", "token")
-
-    fieldsets = (
-        (
-            "Token",
-            {
-                "fields": ("token", "expires"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Owner",
-            {
-                "fields": ("created_by",),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": ("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
-    )
-
-    list_filter = ("created_by",)
-    ordering = ["-created_at"]
-    list_per_page = 100
-
-
-class OutboundWebhookAdminForm(WebhookModelForm):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.fields["ref"] = forms.ChoiceField(
-            label=self.fields["ref"].label,
-            help_text=self.fields["ref"].help_text,
-            choices=[
-                (ref, f"{capfirst(model_from_reference(ref, check_hooks=False)._meta.verbose_name_plural)} ({ref})")
-                for ref in sorted(webhook_settings.HOOKS)
-            ],
-        )
-
-
-class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
-    form = OutboundWebhookAdminForm
-    list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
-    sort_fields = _webhook_fields("created_at", "created_by", "id", "ref", "endpoint", "last_success", "last_failure")
-    readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
-
-    fieldsets = (
-        (
-            "Webhook",
-            {
-                "fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"),
-                "classes": ("card", "wide"),
-            },
-        ),
-        (
-            "Authentication",
-            {
-                "fields": _webhook_fields("auth_token"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Status",
-            {
-                "fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Owner",
-            {
-                "fields": _webhook_fields("created_by"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": _webhook_fields("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
-    )
-
-    def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:
-        """Preserve WebhookAdmin's auth token filter with Django's current admin signature."""
-        return not lookup.startswith("auth_token") and admin.ModelAdmin.lookup_allowed(self, lookup, value, request)
-
-
-def register_admin(admin_site: admin.AdminSite) -> None:
-    admin_site.register(APIToken, APITokenAdmin)
-    admin_site.register(get_webhook_model(), CustomWebhookAdmin)

archivebox/api/apps.py

@@ -1,19 +1,5 @@
-__package__ = "archivebox.api"
-
 from django.apps import AppConfig
 
 
 class APIConfig(AppConfig):
-    name = "archivebox.api"
-    label = "api"
-
-    def ready(self):
-        from archivebox.api.webhooks import register_delete_webhook_field_loader
-
-        register_delete_webhook_field_loader()
-
-
-def register_admin(admin_site):
-    from archivebox.api.admin import register_admin
-
-    register_admin(admin_site)
+    name = 'api'

archivebox/api/archive.py

@@ -0,0 +1,184 @@
+# archivebox_api.py
+from typing import List, Optional
+from enum import Enum
+from pydantic import BaseModel
+from ninja import Router
+from main import (
+    add,
+    remove,
+    update,
+    list_all,
+    ONLY_NEW,
+)  # Assuming these functions are defined in main.py
+
+
+# Schemas
+
+class StatusChoices(str, Enum):
+    indexed = 'indexed'
+    archived = 'archived'
+    unarchived = 'unarchived'
+    present = 'present'
+    valid = 'valid'
+    invalid = 'invalid'
+    duplicate = 'duplicate'
+    orphaned = 'orphaned'
+    corrupted = 'corrupted'
+    unrecognized = 'unrecognized'
+
+
+class AddURLSchema(BaseModel):
+    urls: List[str]
+    tag: str = ""
+    depth: int = 0
+    update: bool = not ONLY_NEW  # Default to the opposite of ONLY_NEW
+    update_all: bool = False
+    index_only: bool = False
+    overwrite: bool = False
+    init: bool = False
+    extractors: str = ""
+    parser: str = "auto"
+
+
+class RemoveURLSchema(BaseModel):
+    yes: bool = False
+    delete: bool = False
+    before: Optional[float] = None
+    after: Optional[float] = None
+    filter_type: str = "exact"
+    filter_patterns: Optional[List[str]] = None
+
+
+class UpdateSchema(BaseModel):
+    resume: Optional[float] = None
+    only_new: Optional[bool] = None
+    index_only: Optional[bool] = False
+    overwrite: Optional[bool] = False
+    before: Optional[float] = None
+    after: Optional[float] = None
+    status: Optional[StatusChoices] = None
+    filter_type: Optional[str] = 'exact'
+    filter_patterns: Optional[List[str]] = None
+    extractors: Optional[str] = ""
+
+
+class ListAllSchema(BaseModel):
+    filter_patterns: Optional[List[str]] = None
+    filter_type: str = 'exact'
+    status: Optional[StatusChoices] = None
+    after: Optional[float] = None
+    before: Optional[float] = None
+    sort: Optional[str] = None
+    csv: Optional[str] = None
+    json: bool = False
+    html: bool = False
+    with_headers: bool = False
+
+
+# API Router
+router = Router()
+
+
+@router.post("/add", response={200: dict})
+def api_add(request, payload: AddURLSchema):
+    try:
+        result = add(
+            urls=payload.urls,
+            tag=payload.tag,
+            depth=payload.depth,
+            update=payload.update,
+            update_all=payload.update_all,
+            index_only=payload.index_only,
+            overwrite=payload.overwrite,
+            init=payload.init,
+            extractors=payload.extractors,
+            parser=payload.parser,
+        )
+        # Currently the add function returns a list of ALL items in the DB, ideally only return new items
+        return {
+            "status": "success",
+            "message": "URLs added successfully.",
+            "result": str(result),
+        }
+    except Exception as e:
+        # Handle exceptions raised by the add function or during processing
+        return {"status": "error", "message": str(e)}
+
+
+@router.post("/remove", response={200: dict})
+def api_remove(request, payload: RemoveURLSchema):
+    try:
+        result = remove(
+            yes=payload.yes,
+            delete=payload.delete,
+            before=payload.before,
+            after=payload.after,
+            filter_type=payload.filter_type,
+            filter_patterns=payload.filter_patterns,
+        )
+        return {
+            "status": "success",
+            "message": "URLs removed successfully.",
+            "result": result,
+        }
+    except Exception as e:
+        # Handle exceptions raised by the remove function or during processing
+        return {"status": "error", "message": str(e)}
+
+
+@router.post("/update", response={200: dict})
+def api_update(request, payload: UpdateSchema):
+    try:
+        result = update(
+            resume=payload.resume,
+            only_new=payload.only_new,
+            index_only=payload.index_only,
+            overwrite=payload.overwrite,
+            before=payload.before,
+            after=payload.after,
+            status=payload.status,
+            filter_type=payload.filter_type,
+            filter_patterns=payload.filter_patterns,
+            extractors=payload.extractors,
+        )
+        return {
+            "status": "success",
+            "message": "Archive updated successfully.",
+            "result": result,
+        }
+    except Exception as e:
+        # Handle exceptions raised by the update function or during processing
+        return {"status": "error", "message": str(e)}
+
+
+@router.post("/list_all", response={200: dict})
+def api_list_all(request, payload: ListAllSchema):
+    try:
+        result = list_all(
+            filter_patterns=payload.filter_patterns,
+            filter_type=payload.filter_type,
+            status=payload.status,
+            after=payload.after,
+            before=payload.before,
+            sort=payload.sort,
+            csv=payload.csv,
+            json=payload.json,
+            html=payload.html,
+            with_headers=payload.with_headers,
+        )
+        # TODO: This is kind of bad, make the format a choice field
+        if payload.json:
+            return {"status": "success", "format": "json", "data": result}
+        elif payload.html:
+            return {"status": "success", "format": "html", "data": result}
+        elif payload.csv:
+            return {"status": "success", "format": "csv", "data": result}
+        else:
+            return {
+                "status": "success",
+                "message": "List generated successfully.",
+                "data": result,
+            }
+    except Exception as e:
+        # Handle exceptions raised by the list_all function or during processing
+        return {"status": "error", "message": str(e)}

archivebox/api/auth.py

@@ -1,137 +1,48 @@
-__package__ = "archivebox.api"
-
-from datetime import timedelta
-
-from django.utils import timezone
-from django.http import HttpRequest
 from django.contrib.auth import authenticate
-from django.contrib.auth.models import User
+from ninja import Form, Router, Schema
+from ninja.security import HttpBearer
 
-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader
-from ninja.errors import HttpError
+from api.models import Token
+
+router = Router()
 
 
-def get_or_create_api_token(user: User | None):
-    from archivebox.api.models import APIToken
-
-    if user and user.is_superuser:
-        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
-        if api_tokens.exists():
-            # unexpired token exists, use it
-            api_token = api_tokens.last()
-        else:
-            # does not exist, create a new one
-            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
-
-        if api_token is None:
-            return None
-        assert api_token.is_valid(), f"API token is not valid {api_token}"
-
-        return api_token
-    return None
-
-
-def auth_using_token(token: str | None, request: HttpRequest | None = None) -> User | None:
-    """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
-    from archivebox.api.models import APIToken  # lazy import model to avoid loading it at urls.py import time
-
-    user: User | None = None
-
-    submitted_empty_form = str(token).strip() in ("string", "", "None", "null")
-    if not submitted_empty_form:
+class GlobalAuth(HttpBearer):
+    def authenticate(self, request, token):
         try:
-            api_token = APIToken.objects.get(token=token)
-            if api_token.is_valid() and isinstance(api_token.created_by, User):
-                user = api_token.created_by
-                if request is not None:
-                    setattr(request, "_api_token", api_token)
-        except APIToken.DoesNotExist:
+            return Token.objects.get(token=token).user
+        except Token.DoesNotExist:
             pass
 
-    return user
+
+class AuthSchema(Schema):
+    email: str
+    password: str
 
 
-def token_from_request(request: HttpRequest) -> str:
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-    return token
+@router.post("/authenticate", auth=None)  # overriding global auth
+def get_token(request, auth_data: AuthSchema):
+    user = authenticate(username=auth_data.email, password=auth_data.password)
+    if user:
+        # Assuming a user can have multiple tokens and you want to create a new one every time
+        new_token = Token.objects.create(user=user)
+        return {"token": new_token.token, "expires": new_token.expiry_as_iso8601}
+    else:
+        return {"error": "Invalid credentials"}
 
 
-def authenticated_user_from_request(request: HttpRequest) -> User | None:
-    user = request.user
-    if user.is_authenticated and user.is_active:
-        return user
-
-    token = token_from_request(request)
-    token_user = auth_using_token(token=token, request=request) if token else None
-    if token_user and token_user.is_active:
-        request.user = token_user
-        return token_user
-    return None
+class TokenValidationSchema(Schema):
+    token: str
 
 
-def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
-    """Given a username and password, check if they are valid and return the corresponding user"""
-    user: User | None = None
-
-    submitted_empty_form = (username, password) in (("string", "string"), ("", ""), (None, None))
-    if not submitted_empty_form:
-        authenticated_user = authenticate(
-            username=username,
-            password=password,
-        )
-        if isinstance(authenticated_user, User):
-            user = authenticated_user
-    return user
-
-
-### Base Auth Types
-
-
-def _require_superuser(user: User | None, request: HttpRequest, auth_method: str) -> User | None:
-    if user and user.pk:
-        request.user = user
-        setattr(request, "_api_auth_method", auth_method)
-        if not user.is_superuser:
-            raise HttpError(403, "Valid credentials but User does not have permission (make sure user.is_superuser=True)")
-    return user
-
-
-### Django-Ninja-Provided Auth Methods
-
-
-class HeaderTokenAuth(APIKeyHeader):
-    """Allow authenticating by passing X-API-Key=xyz as a request header"""
-
-    param_name = "X-ArchiveBox-API-Key"
-
-    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
-        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
-
-
-class BearerTokenAuth(HttpBearer):
-    """Allow authenticating by passing Bearer=xyz as a request header"""
-
-    def authenticate(self, request: HttpRequest, token: str) -> User | None:
-        return _require_superuser(auth_using_token(token=token, request=request), request, self.__class__.__name__)
-
-
-class QueryParamTokenAuth(APIKeyQuery):
-    """Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
-
-    param_name = "api_key"
-
-    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
-        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
-
-
-### Enabled Auth Methods
-
-API_AUTH_METHODS = [
-    HeaderTokenAuth(),
-    BearerTokenAuth(),
-    QueryParamTokenAuth(),
-    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
-]
+@router.post("/validate_token", auth=None) # No authentication required for this endpoint
+def validate_token(request, token_data: TokenValidationSchema):
+    try:
+        # Attempt to authenticate using the provided token
+        user = GlobalAuth().authenticate(request, token_data.token)
+        if user:
+            return {"status": "valid"}
+        else:
+            return {"status": "invalid"}
+    except Token.DoesNotExist:
+        return {"status": "invalid"}

archivebox/api/middleware.py

@@ -1,32 +0,0 @@
-__package__ = "archivebox.api"
-
-from django.http import HttpResponse
-
-
-class ApiCorsMiddleware:
-    """Attach permissive CORS headers for API routes (token-based auth)."""
-
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        if request.path.startswith("/api/"):
-            if request.method == "OPTIONS" and request.META.get("HTTP_ACCESS_CONTROL_REQUEST_METHOD"):
-                response = HttpResponse(status=204)
-                return self._add_cors_headers(request, response)
-
-            response = self.get_response(request)
-            return self._add_cors_headers(request, response)
-
-        return self.get_response(request)
-
-    def _add_cors_headers(self, request, response):
-        origin = request.META.get("HTTP_ORIGIN")
-        if not origin:
-            return response
-
-        response["Access-Control-Allow-Origin"] = "*"
-        response["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
-        response["Access-Control-Allow-Headers"] = "Authorization, X-ArchiveBox-API-Key, Content-Type, X-CSRFToken"
-        response["Access-Control-Max-Age"] = "600"
-        return response

archivebox/api/migrations/0001_initial.py

@@ -1,281 +1,28 @@
-# Generated by hand on 2025-12-29
-# Creates APIToken and OutboundWebhook tables using raw SQL
+# Generated by Django 3.1.14 on 2024-04-09 18:52
 
+import api.models
+from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
-import django.utils.timezone
-from django.conf import settings
-from archivebox.uuid_compat import uuid7
-from archivebox.base_models.models import get_or_create_system_user_pk
-import archivebox.api.models
-import signal_webhooks.fields
-import signal_webhooks.utils
-
-
-# Raw sqlite DDL kept EXACTLY as-is; executed only on sqlite via Django's
-# RunSQL (identical statement splitting → byte-for-byte identical behavior).
-INITIAL_SQL = """
-                -- Create api_apitoken table
-                CREATE TABLE IF NOT EXISTS api_apitoken (
-                    id TEXT PRIMARY KEY NOT NULL,
-                    created_at DATETIME NOT NULL,
-                    modified_at DATETIME NOT NULL,
-
-                    token VARCHAR(32) NOT NULL UNIQUE,
-                    expires DATETIME,
-
-                    created_by_id INTEGER NOT NULL,
-
-                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
-                );
-                CREATE INDEX IF NOT EXISTS api_apitoken_created_by_id_idx ON api_apitoken(created_by_id);
-                CREATE INDEX IF NOT EXISTS api_apitoken_created_at_idx ON api_apitoken(created_at);
-                CREATE INDEX IF NOT EXISTS api_apitoken_token_idx ON api_apitoken(token);
-
-                -- Create api_outboundwebhook table
-                CREATE TABLE IF NOT EXISTS api_outboundwebhook (
-                    id TEXT PRIMARY KEY NOT NULL,
-                    created_at DATETIME NOT NULL,
-                    modified_at DATETIME NOT NULL,
-
-                    name VARCHAR(255) NOT NULL UNIQUE,
-                    signal VARCHAR(255) NOT NULL,
-                    ref VARCHAR(1024) NOT NULL,
-                    endpoint VARCHAR(2048) NOT NULL,
-                    headers TEXT NOT NULL DEFAULT '{}',
-                    auth_token TEXT NOT NULL DEFAULT '',
-                    enabled BOOLEAN NOT NULL DEFAULT 1,
-                    keep_last_response BOOLEAN NOT NULL DEFAULT 0,
-                    created DATETIME NOT NULL,
-                    updated DATETIME NOT NULL,
-                    last_response TEXT NOT NULL DEFAULT '',
-                    last_success DATETIME,
-                    last_failure DATETIME,
-
-                    created_by_id INTEGER NOT NULL,
-
-                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
-                );
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_by_id_idx ON api_outboundwebhook(created_by_id);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
-                CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
-                    """
-
-INITIAL_REVERSE_SQL = """
-                DROP TABLE IF EXISTS api_outboundwebhook;
-                DROP TABLE IF EXISTS api_apitoken;
-                    """
-
-
-def _run_sqlite_only_sql(apps, schema_editor):
-    if schema_editor.connection.vendor != "sqlite":
-        return
-    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_forwards(
-        "api",
-        schema_editor,
-        None,
-        None,
-    )
-
-
-def _run_sqlite_only_sql_reverse(apps, schema_editor):
-    if schema_editor.connection.vendor != "sqlite":
-        return
-    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_backwards(
-        "api",
-        schema_editor,
-        None,
-        None,
-    )
-
-
-def _pg_sync_schema(apps, schema_editor):
-    from archivebox.misc.db import rebuild_models_from_migration_state
-
-    rebuild_models_from_migration_state(apps, schema_editor, "api", ["APIToken", "OutboundWebhook"])
-
-
-def _pg_drop_schema(apps, schema_editor):
-    from archivebox.misc.db import drop_models_on_postgres
-
-    drop_models_on_postgres(apps, schema_editor, "api", ["OutboundWebhook", "APIToken"])
 
 
 class Migration(migrations.Migration):
+
     initial = True
 
     dependencies = [
-        ("auth", "0012_alter_user_first_name_max_length"),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
     ]
 
     operations = [
-        migrations.SeparateDatabaseAndState(
-            database_operations=[
-                migrations.RunPython(_run_sqlite_only_sql, reverse_code=_run_sqlite_only_sql_reverse),
-            ],
-            state_operations=[
-                migrations.CreateModel(
-                    name="APIToken",
-                    fields=[
-                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
-                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
-                        ("modified_at", models.DateTimeField(auto_now=True)),
-                        ("token", models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
-                        ("expires", models.DateTimeField(blank=True, null=True)),
-                        (
-                            "created_by",
-                            models.ForeignKey(
-                                default=get_or_create_system_user_pk,
-                                on_delete=django.db.models.deletion.CASCADE,
-                                to=settings.AUTH_USER_MODEL,
-                            ),
-                        ),
-                    ],
-                    options={
-                        "verbose_name": "API Key",
-                        "verbose_name_plural": "API Keys",
-                        "app_label": "api",
-                    },
-                ),
-                migrations.CreateModel(
-                    name="OutboundWebhook",
-                    fields=[
-                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
-                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
-                        ("modified_at", models.DateTimeField(auto_now=True)),
-                        (
-                            "name",
-                            models.CharField(db_index=True, help_text="Webhook name.", max_length=255, unique=True, verbose_name="name"),
-                        ),
-                        (
-                            "signal",
-                            models.CharField(
-                                choices=[
-                                    ("CREATE", "Create"),
-                                    ("UPDATE", "Update"),
-                                    ("DELETE", "Delete"),
-                                    ("M2M", "M2M changed"),
-                                    ("CREATE_OR_UPDATE", "Create or Update"),
-                                    ("CREATE_OR_DELETE", "Create or Delete"),
-                                    ("CREATE_OR_M2M", "Create or M2M changed"),
-                                    ("UPDATE_OR_DELETE", "Update or Delete"),
-                                    ("UPDATE_OR_M2M", "Update or M2M changed"),
-                                    ("DELETE_OR_M2M", "Delete or M2M changed"),
-                                    ("CREATE_UPDATE_OR_DELETE", "Create, Update or Delete"),
-                                    ("CREATE_UPDATE_OR_M2M", "Create, Update or M2M changed"),
-                                    ("CREATE_DELETE_OR_M2M", "Create, Delete or M2M changed"),
-                                    ("UPDATE_DELETE_OR_M2M", "Update, Delete or M2M changed"),
-                                    ("CREATE_UPDATE_DELETE_OR_M2M", "Create, Update or Delete, or M2M changed"),
-                                ],
-                                help_text="Signal the webhook fires to.",
-                                max_length=255,
-                                verbose_name="signal",
-                            ),
-                        ),
-                        (
-                            "ref",
-                            models.CharField(
-                                db_index=True,
-                                help_text="Dot import notation to the model the webhook is for.",
-                                max_length=1023,
-                                validators=[signal_webhooks.utils.model_from_reference],
-                                verbose_name="referenced model",
-                            ),
-                        ),
-                        (
-                            "endpoint",
-                            models.URLField(help_text="Target endpoint for this webhook.", max_length=2047, verbose_name="endpoint"),
-                        ),
-                        (
-                            "headers",
-                            models.JSONField(
-                                blank=True,
-                                default=dict,
-                                help_text="Headers to send with the webhook request.",
-                                validators=[signal_webhooks.utils.is_dict],
-                                verbose_name="headers",
-                            ),
-                        ),
-                        (
-                            "auth_token",
-                            signal_webhooks.fields.TokenField(
-                                blank=True,
-                                default="",
-                                help_text="Authentication token to use in an Authorization header.",
-                                max_length=8000,
-                                validators=[signal_webhooks.utils.decode_cipher_key],
-                                verbose_name="authentication token",
-                            ),
-                        ),
-                        ("enabled", models.BooleanField(default=True, help_text="Is this webhook enabled?", verbose_name="enabled")),
-                        (
-                            "keep_last_response",
-                            models.BooleanField(
-                                default=False,
-                                help_text="Should the webhook keep a log of the latest response it got?",
-                                verbose_name="keep last response",
-                            ),
-                        ),
-                        (
-                            "created",
-                            models.DateTimeField(auto_now_add=True, help_text="When the webhook was created.", verbose_name="created"),
-                        ),
-                        (
-                            "updated",
-                            models.DateTimeField(auto_now=True, help_text="When the webhook was last updated.", verbose_name="updated"),
-                        ),
-                        (
-                            "last_response",
-                            models.CharField(
-                                blank=True,
-                                default="",
-                                help_text="Latest response to this webhook.",
-                                max_length=8000,
-                                verbose_name="last response",
-                            ),
-                        ),
-                        (
-                            "last_success",
-                            models.DateTimeField(
-                                default=None,
-                                help_text="When the webhook last succeeded.",
-                                null=True,
-                                verbose_name="last success",
-                            ),
-                        ),
-                        (
-                            "last_failure",
-                            models.DateTimeField(
-                                default=None,
-                                help_text="When the webhook last failed.",
-                                null=True,
-                                verbose_name="last failure",
-                            ),
-                        ),
-                        (
-                            "created_by",
-                            models.ForeignKey(
-                                default=get_or_create_system_user_pk,
-                                on_delete=django.db.models.deletion.CASCADE,
-                                to=settings.AUTH_USER_MODEL,
-                            ),
-                        ),
-                    ],
-                    options={
-                        "verbose_name": "API Outbound Webhook",
-                        "app_label": "api",
-                    },
-                ),
-                migrations.AddConstraint(
-                    model_name="outboundwebhook",
-                    constraint=models.UniqueConstraint(fields=["ref", "endpoint"], name="prevent_duplicate_hooks_api_outboundwebhook"),
-                ),
+        migrations.CreateModel(
+            name='Token',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('token', models.CharField(default=auth.models.hex_uuid, max_length=32, unique=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('expiry', models.DateTimeField(blank=True, null=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tokens', to=settings.AUTH_USER_MODEL)),
             ],
         ),
-        # On non-sqlite backends the raw DDL above is skipped, so the real
-        # schema diverges from Django state. Rebuild both tables (including the
-        # unique constraint) from the post-migration state (no-op on sqlite).
-        migrations.RunPython(_pg_sync_schema, reverse_code=_pg_drop_schema),
     ]

archivebox/api/models.py

@@ -1,54 +1,30 @@
-__package__ = "archivebox.api"
-
-import secrets
-from archivebox.uuid_compat import CompactUUIDField, uuid7
+import uuid
+from datetime import timedelta
 
 from django.conf import settings
 from django.db import models
 from django.utils import timezone
-from django_stubs_ext.db.models import TypedModelMeta
-from signal_webhooks.models import WebhookBase
+from django.utils.translation import gettext_lazy as _
 
-from archivebox.base_models.models import get_or_create_system_user_pk
+def hex_uuid():
+    return uuid.uuid4().hex
 
 
-def generate_secret_token() -> str:
-    return secrets.token_hex(16)
-
-
-class APIToken(models.Model):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-    token = models.CharField(max_length=32, default=generate_secret_token, unique=True)
-    expires = models.DateTimeField(null=True, blank=True)
-
-    class Meta(TypedModelMeta):
-        app_label = "api"
-        verbose_name = "API Key"
-        verbose_name_plural = "API Keys"
-
-    def __str__(self) -> str:
-        return self.token
+class Token(models.Model):
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="tokens"
+    )
+    token = models.CharField(max_length=32, default=hex_uuid, unique=True)
+    created = models.DateTimeField(auto_now_add=True)
+    expiry = models.DateTimeField(null=True, blank=True)
 
     @property
-    def token_redacted(self):
-        return f"************{self.token[-4:]}"
+    def expiry_as_iso8601(self):
+        """Returns the expiry date of the token in ISO 8601 format or a date 100 years in the future if none."""
+        expiry_date = (
+            self.expiry if self.expiry else timezone.now() + timedelta(days=365 * 100)
+        )
+        return expiry_date.isoformat()
 
-    def is_valid(self, for_date=None):
-        return not self.expires or self.expires >= (for_date or timezone.now())
-
-
-class OutboundWebhook(WebhookBase):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-
-    class Meta(WebhookBase.Meta):
-        app_label = "api"
-        verbose_name = "API Outbound Webhook"
-
-    def __str__(self) -> str:
-        return f"[{self.id}] {self.ref} -> {self.endpoint}"
+    def __str__(self):
+        return self.token

archivebox/api/tests.py

@@ -0,0 +1,27 @@
+from django.test import TestCase
+from ninja.testing import TestClient
+from archivebox.api.archive import router as archive_router
+
+class ArchiveBoxAPITestCase(TestCase):
+    def setUp(self):
+        self.client = TestClient(archive_router)
+
+    def test_add_endpoint(self):
+        response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "test"})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["status"], "success")
+
+    def test_remove_endpoint(self):
+        response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["status"], "success")
+
+    def test_update_endpoint(self):
+        response = self.client.post("/update", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()["status"], "success")
+
+    def test_list_all_endpoint(self):
+        response = self.client.post("/list_all", json={})
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue("success" in response.json()["status"])

archivebox/api/urls.py

@@ -1,26 +0,0 @@
-__package__ = "archivebox.api"
-
-from urllib.parse import quote
-
-from django.http import HttpRequest, HttpResponseRedirect
-from django.shortcuts import redirect
-from django.urls import path
-from django.views.generic.base import RedirectView
-
-from archivebox.core.routes_util import build_web_url
-from .v1_api import urls as v1_api_urls
-
-
-def archive_redirect_view(request: HttpRequest, url: str) -> HttpResponseRedirect:
-    if request.META.get("QUERY_STRING"):
-        url = f"{url}?{request.META['QUERY_STRING']}"
-    return redirect(build_web_url(f"/web/{quote(url, safe=':/')}", request=request))
-
-
-urlpatterns = [
-    path("", RedirectView.as_view(url="/api/v1/docs")),
-    path("archive/<path:url>", archive_redirect_view, name="api-archive-redirect"),
-    path("v1/", RedirectView.as_view(url="/api/v1/docs")),
-    path("v1/", v1_api_urls),
-    path("v1", RedirectView.as_view(url="/api/v1/docs")),
-]

archivebox/api/v1_api.py

@@ -1,127 +0,0 @@
-__package__ = "archivebox.api"
-
-
-from io import StringIO
-from functools import lru_cache
-from typing import Any
-from traceback import format_exception
-from contextlib import redirect_stdout, redirect_stderr
-
-from django.http import HttpRequest, HttpResponse
-from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
-from django.contrib.auth.models import User
-
-from ninja import NinjaAPI, Swagger
-from ninja.openapi.schema import OpenAPISchema
-
-# TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
-
-from archivebox.config import VERSION
-from archivebox.config.version import get_COMMIT_HASH
-
-from archivebox.api.auth import API_AUTH_METHODS
-from archivebox.api.models import APIToken
-
-
-COMMIT_HASH = get_COMMIT_HASH() or "unknown"
-
-html_description = f"""
-<h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
-<br/>
-<i><b>WARNING: This API is still in an early development stage and may change!</b></i>
-<br/>
-<ul>
-<li>⬅️ Manage your server: <a href="/admin/api/"><b>Setup API Keys</b></a>, <a href="/admin/">Go to your Server Admin UI</a>, <a href="/">Go to your Snapshots list</a> 
-<li>💬 Ask questions and get help here: <a href="https://zulip.archivebox.io">ArchiveBox Chat Forum</a></li>
-<li>🐞 Report API bugs here: <a href="https://github.com/ArchiveBox/ArchiveBox/issues">Github Issues</a></li>
-<li>📚 ArchiveBox Documentation: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Github Wiki</a></li>
-<li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
-</ul>
-<small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
-"""
-
-
-def register_urls(api: NinjaAPI) -> NinjaAPI:
-    api.add_router("/auth/", "archivebox.api.v1_auth.router")
-    api.add_router("/core/", "archivebox.api.v1_core.router")
-    api.add_router("/crawls/", "archivebox.api.v1_crawls.router")
-    api.add_router("/cli/", "archivebox.api.v1_cli.router")
-    api.add_router("/machine/", "archivebox.api.v1_machine.router")
-    api.add_router("/personas/", "archivebox.api.v1_personas.router")
-    return api
-
-
-class NinjaAPIWithIOCapture(NinjaAPI):
-    def get_openapi_schema(self, *, path_prefix: str | None = None, path_params: dict[str, Any] | None = None) -> OpenAPISchema:
-        prefix = path_prefix if path_prefix is not None else self.get_root_path(path_params or {})
-        return self._schema_for_prefix(prefix)
-
-    @lru_cache(maxsize=8)
-    def _schema_for_prefix(self, path_prefix: str) -> OpenAPISchema:
-        # Routes are registered at startup. Server discovery can reuse their
-        # schema while preserving distinct URL prefixes for mounted instances.
-        return super().get_openapi_schema(path_prefix=path_prefix)
-
-    def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
-        stdout, stderr = StringIO(), StringIO()
-
-        with redirect_stderr(stderr):
-            with redirect_stdout(stdout):
-                setattr(request, "stdout", stdout)
-                setattr(request, "stderr", stderr)
-
-                response = super().create_temporal_response(request)
-
-        # Disable caching of API responses entirely
-        response["Cache-Control"] = "no-store"
-
-        # Add debug stdout and stderr headers to response
-        response["X-ArchiveBox-Stdout"] = stdout.getvalue().replace("\n", "\\n")[:200]
-        response["X-ArchiveBox-Stderr"] = stderr.getvalue().replace("\n", "\\n")[:200]
-        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
-
-        # Add Auth Headers to response
-        api_token_attr = request.__dict__.get("_api_token")
-        api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
-        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
-
-        response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
-        response["X-ArchiveBox-Auth-Expires"] = token_expiry
-        response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
-        response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
-
-        return response
-
-
-api = NinjaAPIWithIOCapture(
-    title="ArchiveBox API",
-    description=html_description,
-    version=VERSION,
-    auth=API_AUTH_METHODS,
-    urls_namespace="api-1",
-    docs=Swagger(settings={"persistAuthorization": True}),
-)
-api = register_urls(api)
-urls = api.urls
-
-
-@api.exception_handler(Exception)
-def generic_exception_handler(request, err):
-    status = 503
-    if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
-        status = 404
-
-    print("".join(format_exception(err)))
-
-    return api.create_response(
-        request,
-        {
-            "succeeded": False,
-            "message": f"{err.__class__.__name__}: {err}",
-            "errors": [
-                "".join(format_exception(err)),
-            ],
-        },
-        status=status,
-    )

archivebox/api/v1_auth.py

@@ -1,106 +0,0 @@
-__package__ = "archivebox.api"
-
-from django.http import HttpRequest
-
-from ninja import Router, Schema
-
-from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token, HeaderTokenAuth, BearerTokenAuth
-
-
-router = Router(tags=["Authentication"], auth=None)
-
-
-class PasswordAuthSchema(Schema):
-    """Schema for a /get_api_token request"""
-
-    username: str | None = None
-    password: str | None = None
-
-
-@router.post(
-    "/get_api_token",
-    auth=None,
-    summary="Generate an API token for a given username & password (or currently logged-in user)",
-)  # auth=None because they are not authed yet
-def get_api_token(request: HttpRequest, auth_data: PasswordAuthSchema):
-    user = auth_using_password(
-        username=auth_data.username,
-        password=auth_data.password,
-        request=request,
-    )
-
-    if user and user.is_superuser:
-        api_token = get_or_create_api_token(user)
-        assert api_token is not None, "Failed to create API token"
-        return {
-            "success": True,
-            "user_id": str(user.pk),
-            "username": user.username,
-            "token": api_token.token,
-            "expires": api_token.expires.isoformat() if api_token.expires else None,
-        }
-
-    return {"success": False, "errors": ["Invalid credentials"]}
-
-
-class TokenAuthSchema(Schema):
-    """Schema for a /check_api_token request"""
-
-    token: str
-
-
-@router.post(
-    "/check_api_token",
-    auth=None,
-    summary="Validate an API token to make sure its valid and non-expired",
-)  # auth=None because they are not authed yet
-def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
-    user = auth_using_token(
-        token=token_data.token,
-        request=request,
-    )
-    if user:
-        return {"success": True, "user_id": str(user.pk)}
-
-    return {"success": False, "user_id": None}
-
-
-@router.post(
-    "/browser_session",
-    auth=[HeaderTokenAuth(), BearerTokenAuth()],
-    summary="Exchange an administrator API key for a normal browser session",
-)
-def browser_session(request: HttpRequest):
-    from django.conf import settings
-    from django.contrib.auth import login
-    from django.http import JsonResponse
-    from django.utils import timezone
-    from ninja.errors import HttpError
-    from archivebox.core.routes_util import get_admin_base_url
-
-    user = request.auth
-    if not user.is_active or not user.is_superuser:
-        raise HttpError(403, "An active administrator is required")
-    login(request, user, backend="django.contrib.auth.backends.ModelBackend")
-    # A browser session must not outlive the key used to establish it.
-    token = request._api_token
-    lifetime = settings.SESSION_COOKIE_AGE
-    if token.expires:
-        lifetime = min(lifetime, max(0, int((token.expires - timezone.now()).total_seconds())))
-    request.session.set_expiry(lifetime)
-    request.session.save()
-    response = JsonResponse(
-        {
-            "admin_url": get_admin_base_url(request=request).rstrip("/") + "/admin/",
-            "cookie": {
-                "name": settings.SESSION_COOKIE_NAME,
-                "value": request.session.session_key,
-                "expires": request.session.get_expiry_date().timestamp(),
-                "secure": settings.SESSION_COOKIE_SECURE,
-            },
-        },
-    )
-    # The native client installs a host-only HttpOnly cookie on the admin origin;
-    # API and admin hosts can differ, so a Set-Cookie on the API host is insufficient.
-    response["Cache-Control"] = "no-store"
-    return response

archivebox/api/v1_cli.py

@@ -1,329 +0,0 @@
-__package__ = "archivebox.api"
-
-import json
-from io import StringIO
-from typing import Any
-from enum import Enum
-
-from django.http import HttpRequest
-
-from ninja import Router, Schema
-from ninja.errors import HttpError
-from pydantic import Field
-
-from archivebox.misc.util import ansi_to_html
-from archivebox.core.models import SnapshotQuerySet
-
-
-# router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=["ArchiveBox CLI Sub-Commands"])
-
-
-# Schemas
-
-JSONType = list[Any] | dict[str, Any] | bool | int | str | None
-FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
-
-
-class CLICommandResponseSchema(Schema):
-    success: bool
-    errors: list[str]
-    result: JSONType
-    result_format: str = "str"
-    stdout: str
-    stderr: str
-
-
-FilterTypeChoices = Enum(
-    "FilterTypeChoices",
-    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
-    type=str,
-)
-
-
-class AddCommandSchema(Schema):
-    urls: list[str]
-    snapshot_ids: list[str] | None = None
-    tag: str = ""
-    depth: int = 0
-    max_urls: int = 0
-    crawl_max_size: int = 0
-    crawl_timeout: int = 0
-    snapshot_max_size: int = 0
-    parser: str = "auto"
-    plugins: str = ""
-    persona: str = "Default"
-    only_new: bool | None = None
-    update: bool = False
-    overwrite: bool = False
-    index_only: bool = False
-
-
-class SnapshotFilterCommandSchema(Schema):
-    after: float | None = 0
-    before: float | None = None
-    filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
-    status: str | None = None
-    url__icontains: str | None = None
-    url__istartswith: str | None = None
-    tag: str | None = None
-    crawl_id: str | None = None
-    limit: int | None = None
-    sort: str | None = None
-    search: str | None = None
-
-
-class UpdateCommandSchema(SnapshotFilterCommandSchema):
-    resume: str | None = None
-    batch_size: int = 100
-    continuous: bool = False
-    index_only: bool = False
-    migrate_only: bool = False
-
-
-class ScheduleCommandSchema(Schema):
-    import_path: str | None = None
-    add: bool = False
-    show: bool = False
-    foreground: bool = False
-    run_all: bool = False
-    quiet: bool = False
-    every: str | None = None
-    tag: str = ""
-    depth: int = 0
-    only_new: bool | None = None
-    update: bool = False
-    overwrite: bool = False
-    clear: bool = False
-
-
-class ListCommandSchema(SnapshotFilterCommandSchema):
-    as_json: bool = True
-    as_html: bool = False
-    as_csv: str | None = "timestamp,url"
-    with_headers: bool = False
-
-
-class RemoveCommandSchema(SnapshotFilterCommandSchema):
-    filter_type: str = FilterTypeChoices.exact
-    timeout: float = 60.0
-
-
-def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
-    kwargs = args.dict()
-    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
-    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
-    return kwargs
-
-
-@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
-def cli_add(request: HttpRequest, args: AddCommandSchema):
-    from archivebox.cli.archivebox_add import add
-    from archivebox.misc.util import validate_url
-
-    config_overrides: dict[str, object] = {}
-    if args.only_new is not None:
-        config_overrides["ONLY_NEW"] = bool(args.only_new)
-    if args.update or args.overwrite:
-        config_overrides["ONLY_NEW"] = False
-    submitted_urls: str | list[str] = args.urls
-    if len(args.urls) == 1:
-        try:
-            validate_url(args.urls[0])
-        except ValueError:
-            submitted_urls = args.urls[0]
-    else:
-        submitted_urls = []
-        for url in args.urls:
-            try:
-                submitted_urls.append(validate_url(url))
-            except ValueError:
-                continue
-        if not submitted_urls:
-            raise HttpError(400, "No valid URLs were submitted")
-    crawl, snapshots = add(
-        urls=submitted_urls,
-        snapshot_ids=args.snapshot_ids,
-        tag=args.tag,
-        depth=args.depth,
-        max_urls=args.max_urls,
-        crawl_max_size=args.crawl_max_size,
-        crawl_timeout=args.crawl_timeout,
-        snapshot_max_size=args.snapshot_max_size,
-        index_only=args.index_only,
-        plugins=args.plugins,
-        persona=args.persona,
-        parser=args.parser,
-        bg=True,  # Always run in background for API calls
-        created_by_id=request.user.pk,
-        config=config_overrides or None,
-    )
-
-    snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
-    result_payload = {
-        "crawl_id": str(crawl.id),
-        "num_snapshots": len(snapshot_ids),
-        "snapshot_ids": snapshot_ids,
-        "queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
-    }
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-
-    return {
-        "success": True,
-        "errors": [],
-        "result": result_payload,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
-def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
-    from archivebox.core.snapshot_status import normalize_snapshot_status
-
-    try:
-        status = normalize_snapshot_status(args.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
-    update_kwargs["status"] = status
-    update_kwargs["stop_daemon_stack"] = False
-
-    is_filtered_update = any(
-        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
-    )
-    matched_snapshot_ids = []
-    if is_filtered_update:
-        matched_snapshot_ids = [
-            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
-        ]
-
-    update(**update_kwargs)
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-    return {
-        "success": True,
-        "errors": [],
-        "result": {
-            "matched_count": len(matched_snapshot_ids),
-            "snapshot_ids": matched_snapshot_ids,
-        }
-        if is_filtered_update
-        else None,
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
-def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
-    from archivebox.cli.archivebox_schedule import schedule
-
-    config_overrides: dict[str, object] = {}
-    if args.only_new is not None:
-        config_overrides["ONLY_NEW"] = bool(args.only_new)
-    if args.update or args.overwrite:
-        config_overrides["ONLY_NEW"] = False
-    result = schedule(
-        import_path=args.import_path,
-        add=args.add,
-        show=args.show,
-        foreground=args.foreground,
-        run_all=args.run_all,
-        quiet=args.quiet,
-        clear=args.clear,
-        every=args.every,
-        tag=args.tag,
-        depth=args.depth,
-        config=config_overrides or None,
-    )
-
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
-def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
-
-    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
-    as_json = search_kwargs.pop("as_json")
-    as_html = search_kwargs.pop("as_html")
-    as_csv = search_kwargs.pop("as_csv")
-    with_headers = search_kwargs.pop("with_headers")
-    try:
-        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    result_format = "txt"
-    if as_json:
-        result_format = "json"
-        result = [
-            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
-            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
-        ]
-    elif as_html:
-        result_format = "html"
-        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
-    elif as_csv:
-        result_format = "csv"
-        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
-        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
-        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
-    else:
-        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
-
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-    return {
-        "success": True,
-        "errors": [],
-        "result": result,
-        "result_format": result_format,
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
-def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
-    from archivebox.cli.archivebox_remove import remove
-    from archivebox.core.models import Snapshot
-
-    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
-    timeout_arg = remove_kwargs.pop("timeout")
-    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
-    try:
-        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    result = remove(
-        yes=True,  # no way to interactively ask for confirmation via API, so we force yes
-        snapshots=snapshots_to_remove,
-        timeout=timeout,
-    )
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-    return {
-        "success": bool(result["success"]),
-        "errors": [str(result["error"])] if result["error"] else [],
-        "result": result,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }

archivebox/api/v1_crawls.py

@@ -1,278 +0,0 @@
-__package__ = "archivebox.api"
-
-from pathlib import Path
-from uuid import UUID
-from datetime import datetime
-from django.http import FileResponse, HttpRequest
-from django.shortcuts import redirect
-from django.utils import timezone
-
-from django.contrib.auth import get_user_model
-from django.contrib.auth.models import User
-
-from ninja import Router, Schema
-from ninja.errors import HttpError
-
-from archivebox.core.models import Snapshot
-from archivebox.core.permissions import (
-    PERMISSIONS_PUBLIC,
-    PERMISSIONS_UNLISTED,
-    is_admin_user,
-    normalize_permissions,
-)
-from archivebox.config.common import get_config
-from archivebox.crawls.models import Crawl
-from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url
-
-from .auth import API_AUTH_METHODS, authenticated_user_from_request
-
-router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
-
-
-class CrawlSchema(Schema):
-    TYPE: str = "crawls.models.Crawl"
-
-    id: UUID
-
-    modified_at: datetime
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-
-    status: str
-    retry_at: datetime | None
-    is_paused: bool
-
-    urls: str
-    max_depth: int
-    tags_str: str
-    config: dict
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by_id)
-
-    @staticmethod
-    def resolve_created_by_username(obj):
-        user_model = get_user_model()
-        user = user_model.objects.get(id=obj.created_by_id)
-        username = user.username
-        return username if isinstance(username, str) else str(user)
-
-    @staticmethod
-    def resolve_config(obj):
-        # Redact credential values so REST responses can never leak the raw
-        # token/secret/api-key that the operator stored in Crawl.config.
-        from archivebox.config.common import redact_sensitive_config
-
-        return redact_sensitive_config(obj.config)
-
-    @staticmethod
-    def resolve_snapshots(obj, context):
-        if bool(context["request"].__dict__.get("with_snapshots", False)):
-            return obj.snapshot_set.all().distinct()
-        return Snapshot.objects.none()
-
-
-class CrawlUpdateSchema(Schema):
-    action: str | None = None
-    status: str | None = None
-    retry_at: datetime | None = None
-    tags: list[str] | None = None
-    tags_str: str | None = None
-
-
-class CrawlCreateSchema(Schema):
-    urls: list[str]
-    max_depth: int = 0
-    tags: list[str] | None = None
-    tags_str: str = ""
-    label: str = ""
-    notes: str = ""
-    config: dict = {}
-
-
-class CrawlDeleteResponseSchema(Schema):
-    success: bool
-    crawl_id: str
-    deleted_count: int
-    deleted_snapshots: int
-
-
-def normalize_tag_list(tags: list[str] | None = None, tags_str: str = "") -> list[str]:
-    if tags is not None:
-        return [tag.strip() for tag in tags if tag and tag.strip()]
-    return [tag.strip() for tag in tags_str.split(",") if tag.strip()]
-
-
-@router.get("/crawls", response=list[CrawlSchema], url_name="get_crawls")
-def get_crawls(request: HttpRequest):
-    return Crawl.objects.all().distinct()
-
-
-@router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
-def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
-    try:
-        urls = [validate_url(url) for url in data.urls if url and url.strip()]
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    if not urls:
-        raise HttpError(400, "At least one URL is required")
-    if data.max_depth not in (0, 1, 2, 3, 4):
-        raise HttpError(400, "max_depth must be between 0 and 4")
-
-    tags = normalize_tag_list(data.tags, data.tags_str)
-    config = dict(data.config or {})
-    config.setdefault("PARSER", "url_list")
-    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
-    crawl = Crawl.objects.create(
-        urls="\n".join(urls),
-        max_depth=data.max_depth,
-        tags_str=",".join(tags),
-        label=data.label,
-        notes=data.notes,
-        config=config,
-        status=Crawl.StatusChoices.QUEUED,
-        retry_at=timezone.now(),
-        created_by=request.user if isinstance(request.user, User) else None,
-    )
-    return crawl
-
-
-def get_crawl_by_ref(crawl_id: str):
-    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
-
-
-@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
-def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
-    """Get a specific Crawl by id."""
-    setattr(request, "with_snapshots", with_snapshots)
-    setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = get_crawl_by_ref(crawl_id)
-
-    if crawl and as_rss:
-        query = request.GET.copy()
-        query.pop("as_rss", None)
-        query["crawl_id"] = str(crawl.id)
-        return redirect(f"/api/v1/core/snapshots.rss?{query.urlencode()}")
-
-    return crawl
-
-
-def crawl_file(request: HttpRequest, crawl_id: str, path: str):
-    # Try to resolve the crawl first; if it doesn't exist, return 404.
-    try:
-        crawl = get_crawl_by_ref(crawl_id)
-    except Crawl.DoesNotExist:
-        raise HttpError(404, "Crawl not found")
-
-    user = authenticated_user_from_request(request)
-
-    # Gate access using the same model as SnapshotView/can_view_snapshot:
-    # admins always pass; owners can see their own crawls; otherwise the crawl
-    # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
-    if not is_admin_user(request):
-        permissions = normalize_permissions(crawl.permissions)
-        is_owner = bool(user and crawl.created_by_id == user.id)
-        if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
-            raise HttpError(404, "Crawl not found")
-
-    crawl_root = Path(crawl.output_dir).resolve()
-    file_path = (crawl_root / path).resolve()
-    if not file_path.is_file() or crawl_root not in file_path.parents:
-        raise HttpError(404, "Crawl file not found")
-
-    if path == "chrome_screencast/latest.jpg" and request.GET.get("after"):
-        try:
-            started_at = float(request.GET["after"])
-        except ValueError:
-            raise HttpError(400, "Invalid screencast start time")
-        if file_path.stat().st_mtime < started_at:
-            raise HttpError(404, "No frame for this capture yet")
-
-    response = FileResponse(file_path.open("rb"))
-    response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
-    response["Pragma"] = "no-cache"
-    response["Expires"] = "0"
-    response["X-Content-Type-Options"] = "nosniff"
-    return response
-
-
-@router.get("/crawl/{crawl_id}/files/{filename}", auth=None, url_name="crawl_file_root")
-def crawl_file_root(request: HttpRequest, crawl_id: str, filename: str):
-    return crawl_file(request, crawl_id, filename)
-
-
-@router.get("/crawl/{crawl_id}/files/{folder}/{filename}", auth=None, url_name="crawl_file_nested_1")
-def crawl_file_nested_1(request: HttpRequest, crawl_id: str, folder: str, filename: str):
-    return crawl_file(request, crawl_id, f"{folder}/{filename}")
-
-
-@router.get("/crawl/{crawl_id}/files/{folder}/{subfolder}/{filename}", auth=None, url_name="crawl_file_nested_2")
-def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfolder: str, filename: str):
-    return crawl_file(request, crawl_id, f"{folder}/{subfolder}/{filename}")
-
-
-@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
-def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
-    """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = get_crawl_by_ref(crawl_id)
-    payload = data.dict(exclude_unset=True)
-    update_fields = ["modified_at"]
-
-    action = payload.pop("action", None)
-    if action:
-        if action == "pause":
-            crawl.pause()
-            return crawl
-        if action in ("resume", "unpause"):
-            crawl.resume()
-            return crawl
-        if action == "cancel":
-            crawl.cancel()
-            return crawl
-        raise HttpError(400, f"Invalid action: {action}")
-
-    tags = payload.pop("tags", None)
-    tags_str = payload.pop("tags_str", None)
-    if tags is not None or tags_str is not None:
-        crawl.tags_str = ",".join(normalize_tag_list(tags, tags_str or ""))
-        update_fields.append("tags_str")
-
-    if "status" in payload:
-        if payload["status"] not in Crawl.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        if payload["status"] == Crawl.StatusChoices.SEALED:
-            crawl.cancel()
-            return crawl
-        crawl.status = payload["status"]
-        update_fields.append("status")
-
-    if "retry_at" in payload:
-        crawl.retry_at = payload["retry_at"]
-        update_fields.append("retry_at")
-
-    crawl.save(update_fields=update_fields)
-    return crawl
-
-
-@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
-def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = get_crawl_by_ref(crawl_id)
-    crawl_id_str = str(crawl.id)
-    crawl.cancel()
-
-    from archivebox.crawls.locks import crawl_lifecycle_lock
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        crawl = get_crawl_by_ref(crawl_id_str)
-        snapshot_count = crawl.snapshot_set.count()
-        deleted_count, _ = crawl.delete()
-    return {
-        "success": True,
-        "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
-        "deleted_snapshots": snapshot_count,
-    }

archivebox/api/v1_machine.py

@@ -1,161 +0,0 @@
-__package__ = "archivebox.api"
-
-from uuid import UUID
-from typing import Annotated
-from datetime import datetime
-
-from django.http import HttpRequest
-
-from ninja import FilterLookup, FilterSchema, Query, Router, Schema
-from ninja.pagination import paginate
-
-from archivebox.api.v1_core import CustomPagination
-
-
-router = Router(tags=["Machine and Dependencies"])
-
-
-# ============================================================================
-# Machine Schemas
-# ============================================================================
-
-
-class MachineSchema(Schema):
-    """Schema for Machine model."""
-
-    TYPE: str = "machine.Machine"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    guid: str
-    hostname: str
-    hw_in_docker: bool
-    hw_in_vm: bool
-    hw_manufacturer: str
-    hw_product: str
-    hw_uuid: str
-    os_arch: str
-    os_family: str
-    os_platform: str
-    os_release: str
-    os_kernel: str
-    stats: dict
-    num_uses_succeeded: int
-    num_uses_failed: int
-
-
-class MachineFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup("id__startswith")] = None
-    hostname: Annotated[str | None, FilterLookup("hostname__icontains")] = None
-    os_platform: Annotated[str | None, FilterLookup("os_platform__icontains")] = None
-    os_arch: Annotated[str | None, FilterLookup("os_arch")] = None
-    hw_in_docker: Annotated[bool | None, FilterLookup("hw_in_docker")] = None
-    hw_in_vm: Annotated[bool | None, FilterLookup("hw_in_vm")] = None
-    bin_providers: Annotated[str | None, FilterLookup("bin_providers__icontains")] = None
-
-
-# ============================================================================
-# Binary Schemas
-# ============================================================================
-
-
-class BinarySchema(Schema):
-    """Schema for Binary model."""
-
-    TYPE: str = "machine.Binary"
-    id: UUID
-    created_at: datetime
-    modified_at: datetime
-    machine_id: UUID
-    machine_hostname: str
-    name: str
-    binproviders: str
-    binprovider: str
-    abspath: str
-    version: str
-    sha256: str
-    status: str
-    is_valid: bool
-    num_uses_succeeded: int
-    num_uses_failed: int
-
-    @staticmethod
-    def resolve_machine_hostname(obj) -> str:
-        return obj.machine.hostname
-
-    @staticmethod
-    def resolve_is_valid(obj) -> bool:
-        return obj.is_valid
-
-
-class BinaryFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup("id__startswith")] = None
-    name: Annotated[str | None, FilterLookup("name__icontains")] = None
-    binprovider: Annotated[str | None, FilterLookup("binprovider")] = None
-    status: Annotated[str | None, FilterLookup("status")] = None
-    machine_id: Annotated[str | None, FilterLookup("machine_id__startswith")] = None
-    version: Annotated[str | None, FilterLookup("version__icontains")] = None
-
-
-# ============================================================================
-# Machine Endpoints
-# ============================================================================
-
-
-@router.get("/machines", response=list[MachineSchema], url_name="get_machines")
-@paginate(CustomPagination)
-def get_machines(request: HttpRequest, filters: Query[MachineFilterSchema]):
-    """List all machines."""
-    from archivebox.machine.models import Machine
-
-    return filters.filter(Machine.objects.all()).distinct()
-
-
-@router.get("/machine/current", response=MachineSchema, url_name="get_current_machine")
-def get_current_machine(request: HttpRequest):
-    """Get the current machine."""
-    from archivebox.machine.models import Machine
-
-    return Machine.current()
-
-
-@router.get("/machine/{machine_id}", response=MachineSchema, url_name="get_machine")
-def get_machine(request: HttpRequest, machine_id: str):
-    """Get a specific machine by ID."""
-    from archivebox.machine.models import Machine
-    from django.db.models import Q
-
-    return Machine.objects.get(Q(id__startswith=machine_id) | Q(hostname__iexact=machine_id))
-
-
-# ============================================================================
-
-
-# ============================================================================
-# Binary Endpoints
-# ============================================================================
-
-
-@router.get("/binaries", response=list[BinarySchema], url_name="get_binaries")
-@paginate(CustomPagination)
-def get_binaries(request: HttpRequest, filters: Query[BinaryFilterSchema]):
-    """List all binaries."""
-    from archivebox.machine.models import Binary
-
-    return filters.filter(Binary.objects.all().select_related("machine")).distinct()
-
-
-@router.get("/binary/{binary_id}", response=BinarySchema, url_name="get_binary")
-def get_binary(request: HttpRequest, binary_id: str):
-    """Get a specific binary by ID."""
-    from archivebox.machine.models import Binary
-
-    return Binary.objects.select_related("machine").get(id__startswith=binary_id)
-
-
-@router.get("/binary/by-name/{name}", response=list[BinarySchema], url_name="get_binaries_by_name")
-def get_binaries_by_name(request: HttpRequest, name: str):
-    """Get all binaries with the given name."""
-    from archivebox.machine.models import Binary
-
-    return list(Binary.objects.filter(name__iexact=name).select_related("machine"))

archivebox/api/v1_personas.py

@@ -1,176 +0,0 @@
-__package__ = "archivebox.api"
-
-import json
-from datetime import datetime
-from typing import Any, Literal
-from uuid import UUID
-
-from django.core.exceptions import ValidationError
-from django.db.models import Q
-from django.http import HttpRequest
-from ninja import Router, Schema
-from ninja.errors import HttpError
-from ninja.pagination import paginate
-from pydantic import Field
-
-from archivebox.api.v1_core import CustomPagination
-from archivebox.personas.importers import validate_persona_name
-from archivebox.personas.models import Persona
-
-
-router = Router(tags=["Personas"])
-
-
-class PersonaBrowserSettingsSchema(Schema):
-    user_agent: str = ""
-    viewport_size: str = ""
-    viewport_device_scale_factor: float | None = None
-    color_scheme: Literal["", "light", "dark"] = ""
-    language: str = ""
-    timezone: str = ""
-    geolocation: dict[str, Any] | None = None
-
-
-class PersonaSyncSchema(Schema):
-    extension_persona_id: str
-    name: str
-    settings: PersonaBrowserSettingsSchema = Field(default_factory=PersonaBrowserSettingsSchema)
-    cookies_txt: str = ""
-    auth_json: dict[str, Any] = Field(default_factory=dict)
-
-
-class PersonaSchema(Schema):
-    TYPE: str = "personas.models.Persona"
-    id: UUID
-    name: str
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-    config: dict[str, Any] | None
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by.pk)
-
-    @staticmethod
-    def resolve_created_by_username(obj) -> str:
-        return obj.created_by.username
-
-    @staticmethod
-    def resolve_config(obj):
-        # Redact credential values so REST responses don't leak the raw
-        # token/secret/api-key the operator stored in Persona.config.
-        from archivebox.config.common import redact_sensitive_config
-
-        return redact_sensitive_config(obj.config)
-
-
-class PersonaSyncResponseSchema(Schema):
-    success: bool
-    created: bool
-    persona: PersonaSchema
-    cookies_file_written: bool
-    auth_file_written: bool
-
-
-def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrowserSettingsSchema) -> dict[str, Any]:
-    config: dict[str, Any] = {
-        "BROWSER_EXTENSION_PERSONA_ID": extension_persona_id,
-        "BROWSER_EXTENSION_SYNCED_AT": datetime.utcnow().isoformat() + "Z",
-    }
-
-    if settings.user_agent:
-        config.update(
-            {
-                "USER_AGENT": settings.user_agent,
-                "CHROME_USER_AGENT": settings.user_agent,
-                "WGET_USER_AGENT": settings.user_agent,
-                "CURL_USER_AGENT": settings.user_agent,
-            },
-        )
-    if settings.viewport_size:
-        config.update(
-            {
-                "RESOLUTION": settings.viewport_size,
-                "CHROME_RESOLUTION": settings.viewport_size,
-            },
-        )
-    if settings.viewport_device_scale_factor is not None:
-        config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
-    if "color_scheme" in settings.model_fields_set:
-        config["BROWSER_COLOR_SCHEME"] = settings.color_scheme
-    if settings.language:
-        config["BROWSER_LANGUAGE"] = settings.language
-    if settings.timezone:
-        config["BROWSER_TIMEZONE"] = settings.timezone
-    if settings.geolocation:
-        config["BROWSER_GEOLOCATION"] = settings.geolocation
-
-    return config
-
-
-def find_persona(extension_persona_id: str, name: str) -> Persona | None:
-    named = Persona.find_named(name)
-    return (
-        Persona.objects.filter(Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(pk=named.pk if named else None))
-        .order_by("created_at")
-        .first()
-    )
-
-
-@router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
-@paginate(CustomPagination)
-def get_personas(request: HttpRequest):
-    """List personas available on this ArchiveBox server."""
-    return Persona.objects.all().order_by("name")
-
-
-@router.post("/sync", response=PersonaSyncResponseSchema, url_name="sync_persona")
-def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
-    """
-    Create or update a Persona from a browser extension profile export.
-
-    The extension sends browser settings plus portable auth artifacts. The server
-    keeps browser override settings in Persona.config and writes cookies.txt /
-    auth.json into the persona directory for extractors to consume.
-    """
-    name = payload.name.strip()
-    is_valid, error_message = validate_persona_name(name)
-    if not is_valid:
-        raise ValueError(error_message)
-
-    try:
-        persona = find_persona(payload.extension_persona_id, name)
-        (persona or Persona(name=name)).validate_name(persona.name if persona else name)
-    except ValidationError as err:
-        raise HttpError(409, "; ".join(err.messages)) from err
-    created = persona is None
-    if persona is None:
-        persona = Persona(name=name)
-        if request.user.is_authenticated:
-            persona.created_by = request.user
-
-    persona.config = {
-        **(persona.config or {}),
-        **browser_settings_to_config(payload.extension_persona_id, payload.settings),
-    }
-    persona.save()
-    persona.ensure_dirs()
-
-    cookies_written = False
-    if "cookies_txt" in payload.model_fields_set:
-        (persona.path / "cookies.txt").write_text(payload.cookies_txt)
-        cookies_written = True
-
-    auth_written = False
-    if "auth_json" in payload.model_fields_set:
-        (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
-        auth_written = True
-
-    return {
-        "success": True,
-        "created": created,
-        "persona": persona,
-        "cookies_file_written": cookies_written,
-        "auth_file_written": auth_written,
-    }

archivebox/api/webhooks.py

@@ -1,60 +0,0 @@
-__package__ = "archivebox.api"
-
-from typing import Any
-from collections.abc import Callable
-import logging
-
-from django.db import transaction
-from django.db.models.signals import pre_delete
-from signal_webhooks.handlers import sync_task_handler
-
-
-logger = logging.getLogger(__name__)
-
-
-def load_deferred_fields_for_delete_webhook(sender, instance, using, **kwargs) -> None:
-    """Materialize deferred fields while DELETE webhook rows still exist.
-
-    Admin lists defer large fields, but signal_webhooks serializes all concrete
-    fields in post_delete. Loading one then would query an already-deleted row
-    and drop the event. Hydrate only deferred fields on webhook-enabled models
-    here, preserving the normal list-query optimization.
-    """
-    from signal_webhooks.handlers import find_hook_handler
-    from signal_webhooks.utils import reference_for_model
-
-    ref = reference_for_model(type(instance))
-    deferred_fields = instance.get_deferred_fields()
-    if not deferred_fields or find_hook_handler(ref, "DELETE") is None:
-        return
-
-    instance.refresh_from_db(using=using, fields=deferred_fields)
-
-
-def register_delete_webhook_field_loader() -> None:
-    pre_delete.connect(
-        load_deferred_fields_for_delete_webhook,
-        dispatch_uid="archivebox.load_deferred_fields_for_delete_webhook",
-        weak=False,
-    )
-
-
-def warning_error_handler(hook: Any, error: Exception | None) -> None:
-    if error is not None:
-        logger.warning("Outbound webhook %r failed: %s", hook.name, error)
-        return
-
-    logger.warning("Outbound webhook %r returned a non-success response.", hook.name)
-
-
-def transaction_on_commit_task_handler(hook: Callable[..., None], **kwargs: Any) -> None:
-    def run_webhook() -> None:
-        try:
-            sync_task_handler(hook, **kwargs)
-        except Exception:
-            logger.warning("Outbound webhook failed after transaction commit.", exc_info=True)
-
-    try:
-        transaction.on_commit(run_webhook)
-    except Exception:
-        logger.warning("Could not schedule outbound webhook after transaction commit.", exc_info=True)

archivebox/base_models/__init__.py

@@ -1 +0,0 @@
-__package__ = "archivebox.base_models"

archivebox/base_models/admin.py

@@ -1,930 +0,0 @@
-"""Base admin classes for models using UUIDv7."""
-
-__package__ = "archivebox.base_models"
-
-import json
-import uuid
-from collections.abc import Mapping
-from typing import ClassVar, NotRequired, TypedDict, cast
-
-from django import forms
-from django.contrib import admin
-from django.db import DatabaseError, models
-from django.forms.renderers import BaseRenderer
-from django.http import HttpRequest, QueryDict
-from django.urls import path, register_converter
-from django.utils.safestring import SafeString, mark_safe
-from django_object_actions import DjangoObjectActions
-
-
-class HexUUIDConverter:
-    """URL path converter that canonicalizes UUIDs to their 32-char hex form.
-
-    Accepts both the hyphenated (``aaaaaaaa-bbbb-...``) and bare-hex
-    (``aaaaaaaabbbb...``) UUID strings on the way in (Django's UUIDField
-    parses either), but ``to_url`` always emits the bare-hex form. This is
-    what makes ``reverse("admin:app_model_change", args=[obj.pk])`` produce
-    ``/admin/app/model/06a1a8facb0d.../change/`` instead of the default
-    hyphenated rendering — admin links throughout the app reverse through
-    this converter once ``BaseModelAdmin.get_urls`` swaps in
-    ``<hexuuid:object_id>`` below.
-    """
-
-    regex = r"[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
-
-    def to_python(self, value: str) -> str:
-        # Strip hyphens but stay as a string — Django admin treats the
-        # captured object_id as a string and calls ``model._meta.pk.to_python``
-        # itself, so we don't want to short-circuit that.
-        return value.replace("-", "")
-
-    def to_url(self, value) -> str:
-        if isinstance(value, uuid.UUID):
-            return value.hex
-        return str(value).replace("-", "")
-
-
-register_converter(HexUUIDConverter, "hexuuid")
-
-
-class ConfigOption(TypedDict):
-    plugin: str
-    type: str | list[str]
-    default: object
-    description: str
-    enum: NotRequired[list[object]]
-    pattern: NotRequired[str]
-    minimum: NotRequired[int | float]
-    maximum: NotRequired[int | float]
-
-
-class KeyValueWidget(forms.Widget):
-    """
-    A widget that renders JSON dict as editable key-value input fields
-    with + and - buttons to add/remove rows.
-    Includes autocomplete for available config keys from the plugin system.
-    """
-
-    template_name = ""  # We render manually
-
-    class Media:
-        css: ClassVar[dict[str, list[str]]] = {
-            "all": [],
-        }
-        js: ClassVar[list[str]] = []
-
-    def _get_config_options(self) -> dict[str, ConfigOption]:
-        """Get available config options from plugins."""
-        try:
-            from archivebox.config.common import config_field_metadata
-
-            options: dict[str, ConfigOption] = {}
-            for key, metadata in config_field_metadata().items():
-                option_type = metadata.get("type", "string")
-                option: ConfigOption = {
-                    "plugin": str(metadata.get("plugin", "archivebox")),
-                    "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
-                    "default": metadata.get("default", ""),
-                    "description": str(metadata.get("description", "")),
-                }
-                schema = metadata.get("schema")
-                if isinstance(schema, Mapping):
-                    for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in schema:
-                            option[schema_key] = schema[schema_key]
-                options[key] = option
-            return options
-        except (ImportError, KeyError, TypeError, ValueError):
-            return {}
-
-    def _parse_value(self, value: object) -> dict[str, object]:
-        # Parse JSON value to dict
-        if value is None:
-            return {}
-        if isinstance(value, str):
-            try:
-                parsed = json.loads(value) if value else {}
-            except json.JSONDecodeError:
-                return {}
-            return parsed if isinstance(parsed, dict) else {}
-        if isinstance(value, Mapping):
-            return {str(key): item for key, item in value.items()}
-        return {}
-
-    def render(
-        self,
-        name: str,
-        value: object,
-        attrs: Mapping[str, str] | None = None,
-        renderer: BaseRenderer | None = None,
-    ) -> SafeString:
-        data = self._parse_value(value)
-
-        widget_id = attrs.get("id", name) if attrs else name
-        config_options = self._get_config_options()
-
-        # Build datalist options
-        datalist_options = "\n".join(
-            f'<option value="{self._escape(key)}">{self._escape(opt["description"][:60] or opt["type"])}</option>'
-            for key, opt in sorted(config_options.items())
-        )
-
-        # Build config metadata as JSON for JS
-        config_meta_json = json.dumps(config_options)
-
-        html = f'''
-        <div id="{widget_id}_container" class="key-value-editor" style="width: 100%; max-width: none;">
-            <datalist id="{widget_id}_keys">
-                {datalist_options}
-            </datalist>
-            <div id="{widget_id}_rows" class="key-value-rows">
-        '''
-
-        # Render existing key-value pairs
-        for key, val in data.items():
-            val_str = json.dumps(val) if not isinstance(val, str) else val
-            html += self._render_row(widget_id, key, val_str)
-
-        # Always add one empty row for new entries
-        html += self._render_row(widget_id, "", "")
-
-        html += f'''
-            </div>
-            <div style="display: flex; gap: 8px; align-items: center; margin-top: 8px;">
-                <button type="button" onclick="addKeyValueRow_{widget_id}()"
-                        style="padding: 4px 12px; cursor: pointer; background: #417690; color: white; border: none; border-radius: 4px;">
-                    + Add Row
-                </button>
-            </div>
-            <input type="hidden" name="{name}" id="{widget_id}" value="">
-            <script>
-                (function() {{
-                    var configMeta_{widget_id} = {config_meta_json};
-                    var rowCounter_{widget_id} = 0;
-
-                    function stringifyValue_{widget_id}(value) {{
-                        return typeof value === 'string' ? value : JSON.stringify(value);
-                    }}
-
-                    function getTypes_{widget_id}(meta) {{
-                        if (!meta || meta.type === undefined || meta.type === null) {{
-                            return [];
-                        }}
-                        return Array.isArray(meta.type) ? meta.type : [meta.type];
-                    }}
-
-                    function getMetaForKey_{widget_id}(key) {{
-                        if (!key) {{
-                            return null;
-                        }}
-
-                        var explicitMeta = configMeta_{widget_id}[key];
-                        if (explicitMeta) {{
-                            return Object.assign({{ key: key }}, explicitMeta);
-                        }}
-
-                        if (key.endsWith('_BINARY')) {{
-                            return {{
-                                key: key,
-                                plugin: 'custom',
-                                type: 'string',
-                                default: '',
-                                description: 'Path to binary executable',
-                            }};
-                        }}
-
-                        if (isRegexConfigKey_{widget_id}(key)) {{
-                            return {{
-                                key: key,
-                                plugin: 'custom',
-                                type: 'string',
-                                default: '',
-                                description: 'Regex pattern list',
-                            }};
-                        }}
-
-                        return null;
-                    }}
-
-                    function describeMeta_{widget_id}(meta) {{
-                        if (!meta) {{
-                            return '';
-                        }}
-
-                        var details = '';
-                        if (Array.isArray(meta.enum) && meta.enum.length) {{
-                            details = 'Allowed: ' + meta.enum.map(stringifyValue_{widget_id}).join(', ');
-                        }} else {{
-                            var types = getTypes_{widget_id}(meta);
-                            if (types.length) {{
-                                details = 'Expected: ' + types.join(' or ');
-                            }}
-                        }}
-
-                        if (meta.minimum !== undefined || meta.maximum !== undefined) {{
-                            var bounds = [];
-                            if (meta.minimum !== undefined) bounds.push('min ' + meta.minimum);
-                            if (meta.maximum !== undefined) bounds.push('max ' + meta.maximum);
-                            details += (details ? ' ' : '') + '(' + bounds.join(', ') + ')';
-                        }}
-
-                        return [meta.description || '', details].filter(Boolean).join(' ');
-                    }}
-
-                    function getExampleInput_{widget_id}(key, meta) {{
-                        var types = getTypes_{widget_id}(meta);
-                        if (key.endsWith('_BINARY')) {{
-                            return 'Example: wget or /usr/bin/wget';
-                        }}
-                        if (key.endsWith('_ARGS_EXTRA') || key.endsWith('_ARGS')) {{
-                            return 'Example: ["--extra-arg"]';
-                        }}
-                        if (types.includes('array')) {{
-                            return 'Example: ["value"]';
-                        }}
-                        if (types.includes('object')) {{
-                            return 'Example: {{"key": "value"}}';
-                        }}
-                        return '';
-                    }}
-
-                    function isRegexConfigKey_{widget_id}(key) {{
-                        return key === 'URL_ALLOWLIST' ||
-                            key === 'URL_DENYLIST' ||
-                            key.endsWith('_PATTERN') ||
-                            key.includes('REGEX');
-                    }}
-
-                    function isSimpleFilterPattern_{widget_id}(pattern) {{
-                        return /^[\\w.*:-]+$/.test(pattern);
-                    }}
-
-                    function validateRegexPattern_{widget_id}(pattern) {{
-                        if (!pattern || isSimpleFilterPattern_{widget_id}(pattern)) {{
-                            return '';
-                        }}
-
-                        try {{
-                            new RegExp(pattern);
-                        }} catch (error) {{
-                            return error && error.message ? error.message : 'Invalid regex';
-                        }}
-                        return '';
-                    }}
-
-                    function validateRegexConfig_{widget_id}(key, raw, typeName) {{
-                        if (typeName === 'object') {{
-                            var parsed;
-                            try {{
-                                parsed = JSON.parse(raw);
-                            }} catch (error) {{
-                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
-                            }}
-                            if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {{
-                                return {{ ok: false, value: parsed, message: 'Must be a JSON object' }};
-                            }}
-                            for (var regexKey in parsed) {{
-                                var objectRegexError = validateRegexPattern_{widget_id}(regexKey);
-                                if (objectRegexError) {{
-                                    return {{ ok: false, value: parsed, message: 'Invalid regex key "' + regexKey + '": ' + objectRegexError }};
-                                }}
-                            }}
-                            return {{ ok: true, value: parsed, message: '' }};
-                        }}
-
-                        var patterns = raw.split(/[\\n,]+/).map(function(pattern) {{
-                            return pattern.trim();
-                        }}).filter(Boolean);
-                        for (var i = 0; i < patterns.length; i++) {{
-                            var regexError = validateRegexPattern_{widget_id}(patterns[i]);
-                            if (regexError) {{
-                                return {{ ok: false, value: raw, message: 'Invalid regex "' + patterns[i] + '": ' + regexError }};
-                            }}
-                        }}
-                        return {{ ok: true, value: raw, message: '' }};
-                    }}
-
-                    function validateBinaryValue_{widget_id}(raw) {{
-                        if (!raw) {{
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        if (/['"`]/.test(raw)) {{
-                            return {{ ok: false, value: raw, message: 'Binary paths cannot contain quotes' }};
-                        }}
-
-                        if (/[;&|<>$(){{}}\\[\\]!]/.test(raw)) {{
-                            return {{ ok: false, value: raw, message: 'Binary paths can only be a binary name or absolute path' }};
-                        }}
-
-                        if (raw.startsWith('/')) {{
-                            if (/^[A-Za-z0-9_./+\\- ]+$/.test(raw)) {{
-                                return {{ ok: true, value: raw, message: '' }};
-                            }}
-                            return {{ ok: false, value: raw, message: 'Absolute paths may only contain path-safe characters' }};
-                        }}
-
-                        if (/^[A-Za-z0-9_.+-]+$/.test(raw)) {{
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        return {{ ok: false, value: raw, message: 'Enter a binary name like wget or an absolute path like /usr/bin/wget' }};
-                    }}
-
-                    function parseValue_{widget_id}(raw) {{
-                        try {{
-                            if (raw === 'true') return true;
-                            if (raw === 'false') return false;
-                            if (raw === 'null') return null;
-                            if (raw !== '' && !isNaN(raw)) return Number(raw);
-                            if ((raw.startsWith('{{') && raw.endsWith('}}')) ||
-                                (raw.startsWith('[') && raw.endsWith(']')) ||
-                                (raw.startsWith('"') && raw.endsWith('"'))) {{
-                                return JSON.parse(raw);
-                            }}
-                        }} catch (error) {{
-                            return raw;
-                        }}
-                        return raw;
-                    }}
-
-                    function sameValue_{widget_id}(left, right) {{
-                        return left === right || JSON.stringify(left) === JSON.stringify(right);
-                    }}
-
-                    function parseTypedValue_{widget_id}(raw, typeName, meta) {{
-                        var numberValue;
-                        var parsed;
-
-                        if (typeName && meta && meta.key && isRegexConfigKey_{widget_id}(meta.key)) {{
-                            return validateRegexConfig_{widget_id}(meta.key, raw, typeName);
-                        }}
-
-                        if (typeName === 'string' && meta && meta.key && meta.key.endsWith('_BINARY')) {{
-                            return validateBinaryValue_{widget_id}(raw);
-                        }}
-
-                        if (typeName === 'string') {{
-                            if (meta.pattern) {{
-                                try {{
-                                    if (!(new RegExp(meta.pattern)).test(raw)) {{
-                                        return {{ ok: false, value: raw, message: 'Must match pattern ' + meta.pattern }};
-                                    }}
-                                }} catch (error) {{}}
-                            }}
-                            return {{ ok: true, value: raw, message: '' }};
-                        }}
-
-                        if (typeName === 'integer') {{
-                            if (!/^-?\\d+$/.test(raw)) {{
-                                return {{ ok: false, value: raw, message: 'Must be an integer' }};
-                            }}
-                            numberValue = Number(raw);
-                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
-                            }}
-                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
-                            }}
-                            return {{ ok: true, value: numberValue, message: '' }};
-                        }}
-
-                        if (typeName === 'number') {{
-                            if (raw === '' || isNaN(raw)) {{
-                                return {{ ok: false, value: raw, message: 'Must be a number' }};
-                            }}
-                            numberValue = Number(raw);
-                            if (meta.minimum !== undefined && numberValue < meta.minimum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at least ' + meta.minimum }};
-                            }}
-                            if (meta.maximum !== undefined && numberValue > meta.maximum) {{
-                                return {{ ok: false, value: numberValue, message: 'Must be at most ' + meta.maximum }};
-                            }}
-                            return {{ ok: true, value: numberValue, message: '' }};
-                        }}
-
-                        if (typeName === 'boolean') {{
-                            var lowered = raw.toLowerCase();
-                            if (lowered === 'true' || raw === '1') return {{ ok: true, value: true, message: '' }};
-                            if (lowered === 'false' || raw === '0') return {{ ok: true, value: false, message: '' }};
-                            return {{ ok: false, value: raw, message: 'Must be true or false' }};
-                        }}
-
-                        if (typeName === 'null') {{
-                            return raw === 'null'
-                                ? {{ ok: true, value: null, message: '' }}
-                                : {{ ok: false, value: raw, message: 'Must be null' }};
-                        }}
-
-                        if (typeName === 'array' || typeName === 'object') {{
-                            try {{
-                                parsed = JSON.parse(raw);
-                            }} catch (error) {{
-                                return {{ ok: false, value: raw, message: 'Must be valid JSON' }};
-                            }}
-
-                            if (typeName === 'array' && Array.isArray(parsed)) {{
-                                return {{ ok: true, value: parsed, message: '' }};
-                            }}
-                            if (typeName === 'object' && parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {{
-                                return {{ ok: true, value: parsed, message: '' }};
-                            }}
-
-                            return {{
-                                ok: false,
-                                value: parsed,
-                                message: typeName === 'array' ? 'Must be a JSON array' : 'Must be a JSON object',
-                            }};
-                        }}
-
-                        return {{ ok: true, value: parseValue_{widget_id}(raw), message: '' }};
-                    }}
-
-                    function validateValueAgainstMeta_{widget_id}(raw, meta) {{
-                        if (!meta || raw === '') {{
-                            return {{ state: 'neutral', value: raw, message: '' }};
-                        }}
-
-                        var enumValues = Array.isArray(meta.enum) ? meta.enum : [];
-                        var types = getTypes_{widget_id}(meta);
-                        if (!types.length) {{
-                            types = ['string'];
-                        }}
-
-                        var error = 'Invalid value';
-                        for (var i = 0; i < types.length; i++) {{
-                            var candidate = parseTypedValue_{widget_id}(raw, types[i], meta);
-                            if (!candidate.ok) {{
-                                error = candidate.message || error;
-                                continue;
-                            }}
-                            if (enumValues.length && !enumValues.some(function(enumValue) {{
-                                return sameValue_{widget_id}(enumValue, candidate.value) || stringifyValue_{widget_id}(enumValue) === raw;
-                            }})) {{
-                                error = 'Must be one of: ' + enumValues.map(stringifyValue_{widget_id}).join(', ');
-                                continue;
-                            }}
-                            return {{ state: 'valid', value: candidate.value, message: '' }};
-                        }}
-
-                        return {{ state: 'invalid', value: raw, message: error }};
-                    }}
-
-                    function ensureRowId_{widget_id}(row) {{
-                        if (!row.dataset.rowId) {{
-                            row.dataset.rowId = String(rowCounter_{widget_id}++);
-                        }}
-                        return row.dataset.rowId;
-                    }}
-
-                    function setRowHelp_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var help = row.querySelector('.kv-help');
-                        if (!keyInput || !help) {{
-                            return;
-                        }}
-
-                        var key = keyInput.value.trim();
-                        if (!key) {{
-                            help.textContent = '';
-                            return;
-                        }}
-
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (meta) {{
-                            var extra = isRegexConfigKey_{widget_id}(key)
-                                ? ((meta.type === 'object' || (Array.isArray(meta.type) && meta.type.includes('object')))
-                                    ? ' Expected: JSON object with regex keys.'
-                                    : ' Expected: valid regex.')
-                                : '';
-                            var example = getExampleInput_{widget_id}(key, meta);
-                            help.textContent = [describeMeta_{widget_id}(meta) + extra, example].filter(Boolean).join(' ');
-                        }} else {{
-                            help.textContent = 'Custom key';
-                        }}
-                    }}
-
-                    function configureValueInput_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var valueInput = row.querySelector('.kv-value');
-                        var datalist = row.querySelector('.kv-value-options');
-                        if (!keyInput || !valueInput || !datalist) {{
-                            return;
-                        }}
-
-                        var rowId = ensureRowId_{widget_id}(row);
-                        datalist.id = '{widget_id}_value_options_' + rowId;
-
-                        var meta = getMetaForKey_{widget_id}(keyInput.value.trim());
-                        var enumValues = Array.isArray(meta && meta.enum) ? meta.enum : [];
-                        var types = getTypes_{widget_id}(meta);
-                        if (!enumValues.length && types.includes('boolean')) {{
-                            enumValues = ['True', 'False'];
-                        }}
-                        if (enumValues.length) {{
-                            datalist.innerHTML = enumValues.map(function(enumValue) {{
-                                return '<option value="' + stringifyValue_{widget_id}(enumValue).replace(/"/g, '&quot;') + '"></option>';
-                            }}).join('');
-                            valueInput.setAttribute('list', datalist.id);
-                        }} else {{
-                            datalist.innerHTML = '';
-                            valueInput.removeAttribute('list');
-                        }}
-                    }}
-
-                    function setValueValidationState_{widget_id}(input, state, message) {{
-                        if (!input) {{
-                            return;
-                        }}
-
-                        if (state === 'valid') {{
-                            input.style.borderColor = '#2da44e';
-                            input.style.boxShadow = '0 0 0 1px rgba(45, 164, 78, 0.18)';
-                            input.style.backgroundColor = '#f6ffed';
-                        }} else if (state === 'invalid') {{
-                            input.style.borderColor = '#cf222e';
-                            input.style.boxShadow = '0 0 0 1px rgba(207, 34, 46, 0.18)';
-                            input.style.backgroundColor = '#fff8f8';
-                        }} else {{
-                            input.style.borderColor = '#ccc';
-                            input.style.boxShadow = 'none';
-                            input.style.backgroundColor = '';
-                        }}
-                        input.title = message || '';
-                    }}
-
-                    function applyValueValidation_{widget_id}(row) {{
-                        var keyInput = row.querySelector('.kv-key');
-                        var valueInput = row.querySelector('.kv-value');
-                        if (!keyInput || !valueInput) {{
-                            return;
-                        }}
-
-                        var key = keyInput.value.trim();
-                        if (!key) {{
-                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
-                            return;
-                        }}
-
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (!meta) {{
-                            setValueValidationState_{widget_id}(valueInput, 'neutral', '');
-                            return;
-                        }}
-
-                        var validation = validateValueAgainstMeta_{widget_id}(valueInput.value.trim(), meta);
-                        setValueValidationState_{widget_id}(valueInput, validation.state, validation.message);
-                    }}
-
-                    function coerceValueForStorage_{widget_id}(key, raw) {{
-                        var meta = getMetaForKey_{widget_id}(key);
-                        if (!meta) {{
-                            return parseValue_{widget_id}(raw);
-                        }}
-
-                        var validation = validateValueAgainstMeta_{widget_id}(raw, meta);
-                        return validation.state === 'valid' ? validation.value : raw;
-                    }}
-
-                    function initializeRows_{widget_id}() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
-                            ensureRowId_{widget_id}(row);
-                            configureValueInput_{widget_id}(row);
-                            setRowHelp_{widget_id}(row);
-                            applyValueValidation_{widget_id}(row);
-                        }});
-                    }}
-
-                    function updateHiddenField_{widget_id}() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        var rows = container.querySelectorAll('.key-value-row');
-                        var result = {{}};
-                        rows.forEach(function(row) {{
-                            var keyInput = row.querySelector('.kv-key');
-                            var valInput = row.querySelector('.kv-value');
-                            if (keyInput && valInput && keyInput.value.trim()) {{
-                                var key = keyInput.value.trim();
-                                var val = valInput.value.trim();
-                                result[key] = coerceValueForStorage_{widget_id}(key, val);
-                            }}
-                        }});
-                        document.getElementById('{widget_id}').value = JSON.stringify(result);
-                    }}
-
-                    window.addKeyValueRow_{widget_id} = function() {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        var newRow = document.createElement('div');
-                        newRow.className = 'key-value-row';
-                        newRow.style.cssText = 'margin-bottom: 6px;';
-                        newRow.innerHTML = '<div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">' +
-                            '<input type="text" class="kv-key" placeholder="KEY" list="{widget_id}_keys" ' +
-                            'style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
-                            '<input type="text" class="kv-value" placeholder="value" ' +
-                            'style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">' +
-                            '<datalist class="kv-value-options"></datalist>' +
-                            '<button type="button" onclick="removeKeyValueRow_{widget_id}(this)" ' +
-                            'style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>' +
-                            '</div>' +
-                            '<div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>';
-                        container.appendChild(newRow);
-                        ensureRowId_{widget_id}(newRow);
-                        configureValueInput_{widget_id}(newRow);
-                        setRowHelp_{widget_id}(newRow);
-                        applyValueValidation_{widget_id}(newRow);
-                        updateHiddenField_{widget_id}();
-                        newRow.querySelector('.kv-key').focus();
-                    }};
-
-                    window.removeKeyValueRow_{widget_id} = function(btn) {{
-                        var row = btn.closest('.key-value-row');
-                        row.remove();
-                        updateHiddenField_{widget_id}();
-                    }};
-
-                    window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
-
-                    function configRowForKey_{widget_id}(key) {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        if (!container) {{
-                            return null;
-                        }}
-                        var match = null;
-                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
-                            if (match) {{ return; }}
-                            var keyInput = row.querySelector('.kv-key');
-                            if (keyInput && keyInput.value.trim() === key) {{
-                                match = row;
-                            }}
-                        }});
-                        if (!match) {{
-                            window.addKeyValueRow_{widget_id}();
-                            var rows = container.querySelectorAll('.key-value-row');
-                            match = rows[rows.length - 1];
-                            var keyInput = match.querySelector('.kv-key');
-                            if (keyInput) {{
-                                keyInput.value = key;
-                                keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
-                            }}
-                        }}
-                        return match;
-                    }}
-
-                    function prefillConfigFromQuery_{widget_id}() {{
-                        var params = new URLSearchParams(window.location.search);
-                        var consumedConfigKeys = [];
-                        params.forEach(function(value, key) {{
-                            if (!/^[A-Z][A-Z0-9_]*$/.test(key) || !configMeta_{widget_id}[key]) {{
-                                return;
-                            }}
-                            consumedConfigKeys.push(key);
-                            var match = configRowForKey_{widget_id}(key);
-                            var valueInput = match && match.querySelector('.kv-value');
-                            if (valueInput) {{
-                                valueInput.value = value;
-                                valueInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
-                            }}
-                        }});
-                        updateHiddenField_{widget_id}();
-                        if (consumedConfigKeys.length) {{
-                            var cleanUrl = new URL(window.location.href);
-                            consumedConfigKeys.forEach(function(key) {{ cleanUrl.searchParams.delete(key); }});
-                            window.history.replaceState(null, '', cleanUrl.pathname + cleanUrl.search + cleanUrl.hash);
-                        }}
-                    }}
-
-                    function focusConfigKeyFromHash_{widget_id}() {{
-                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
-                        // to (or creates) the matching row in this editor. Used by
-                        // the setup wizard and live-config detail pages.
-                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
-                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
-                            return;
-                        }}
-                        var match = configRowForKey_{widget_id}(hash);
-                        if (!match) {{
-                            return;
-                        }}
-                        match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
-                        var prevOutline = match.style.outline;
-                        match.style.outline = '2px solid #f59e0b';
-                        match.style.outlineOffset = '2px';
-                        match.style.transition = 'outline 1.2s ease-out';
-                        setTimeout(function() {{
-                            match.style.outline = prevOutline || 'none';
-                        }}, 1400);
-                        var valueInput = match.querySelector('.kv-value');
-                        if (valueInput) {{
-                            valueInput.focus();
-                            try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
-                        }}
-                    }}
-
-                    // Initialize on load
-                    document.addEventListener('DOMContentLoaded', function() {{
-                        initializeRows_{widget_id}();
-                        prefillConfigFromQuery_{widget_id}();
-                        updateHiddenField_{widget_id}();
-                        focusConfigKeyFromHash_{widget_id}();
-                    }});
-                    // Also run immediately in case DOM is already ready
-                    if (document.readyState !== 'loading') {{
-                        initializeRows_{widget_id}();
-                        prefillConfigFromQuery_{widget_id}();
-                        updateHiddenField_{widget_id}();
-                        focusConfigKeyFromHash_{widget_id}();
-                    }}
-
-                    window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
-
-                    // Update on any input change
-                    var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
-
-                    rowsEl_{widget_id}.addEventListener('input', function(event) {{
-                        var row = event.target.closest('.key-value-row');
-                        if (!row) {{
-                            return;
-                        }}
-
-                        if (event.target.classList.contains('kv-key')) {{
-                            configureValueInput_{widget_id}(row);
-                            setRowHelp_{widget_id}(row);
-                        }}
-
-                        if (event.target.classList.contains('kv-key') || event.target.classList.contains('kv-value')) {{
-                            applyValueValidation_{widget_id}(row);
-                            updateHiddenField_{widget_id}();
-                        }}
-                    }});
-                }})();
-            </script>
-        </div>
-        '''
-        return mark_safe(html)
-
-    def _render_row(self, widget_id: str, key: str, value: str) -> str:
-        from archivebox.config.common import is_sensitive_config_key
-
-        # Sensitive keys (``*TOKEN*``, ``*SECRET*``, ``*API_KEY*``, ``*APIKEY*``) are
-        # rendered write-only: the input is a password field with a placeholder
-        # showing the value is set, but the raw value is NEVER sent to the browser.
-        # When the user submits the form with the field left blank, the
-        # ``ConfigEditorMixin.save_model`` hook re-merges the previously-saved
-        # value so leaving it untouched is a no-op rather than a destructive clear.
-        is_sensitive = is_sensitive_config_key(key)
-        has_value = bool(value)
-        if is_sensitive:
-            input_type = "password"
-            rendered_value = ""
-            placeholder = (
-                "•••••• (saved — enter new value to replace, clear by deleting row)" if has_value else "value (will be saved write-only)"
-            )
-            extra_attrs = ' autocomplete="off" data-sensitive="1"' + (' data-had-value="1"' if has_value else "")
-        else:
-            input_type = "text"
-            rendered_value = self._escape(value)
-            placeholder = "value"
-            extra_attrs = ""
-        return f'''
-            <div class="key-value-row" style="margin-bottom: 6px;">
-                <div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
-                    <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
-                           style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <input type="{input_type}" class="kv-value" value="{rendered_value}" placeholder="{self._escape(placeholder)}"{extra_attrs}
-                           style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <datalist class="kv-value-options"></datalist>
-                    <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
-                            style="padding: 4px 10px; cursor: pointer; background: #ba2121; color: white; border: none; border-radius: 4px; font-weight: bold;">−</button>
-                </div>
-                <div class="kv-help" style="margin-top: 4px; font-size: 11px; color: #666; font-style: italic;"></div>
-            </div>
-        '''
-
-    def _escape(self, s: object) -> str:
-        """Escape HTML special chars in attribute values."""
-        if not s:
-            return ""
-        return str(s).replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;")
-
-    def value_from_datadict(
-        self,
-        data: QueryDict | Mapping[str, object],
-        files: object,
-        name: str,
-    ) -> str:
-        value = data.get(name, "{}")
-        return value if isinstance(value, str) else "{}"
-
-
-class ConfigEditorMixin(admin.ModelAdmin):
-    """
-    Mixin for admin classes with a config JSON field.
-
-    Provides a key-value editor widget with autocomplete for available config keys.
-    """
-
-    def formfield_for_dbfield(
-        self,
-        db_field: models.Field,
-        request: HttpRequest,
-        **kwargs: object,
-    ) -> forms.Field | None:
-        """Use KeyValueWidget for the config JSON field."""
-        if db_field.name == "config":
-            kwargs["widget"] = KeyValueWidget()
-        return super().formfield_for_dbfield(db_field, request, **kwargs)
-
-    def save_model(self, request: HttpRequest, obj, form, change):
-        """Preserve write-only redacted credentials on save.
-
-        The KeyValueWidget renders sensitive keys (``*TOKEN*``, ``*SECRET*``,
-        ``*API_KEY*``, ``*APIKEY*``) with an empty value + password input —
-        the real value never leaves the server. On submit, an empty value
-        for a sensitive key that was previously set means "leave untouched",
-        not "clear it." We honor that here by re-merging the stored value
-        before the row is written. Explicitly removing the row in the UI
-        still clears it (the key is gone from the submitted JSON, so there's
-        nothing to merge over).
-        """
-        from archivebox.config.common import is_sensitive_config_key
-
-        if change and obj.pk and obj.config is not None:
-            try:
-                stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
-            except (AttributeError, DatabaseError, TypeError, ValueError):
-                stored = {}
-            if isinstance(stored, dict):
-                new_config = dict(obj.config or {})
-                for key, new_value in list(new_config.items()):
-                    if not is_sensitive_config_key(key):
-                        continue
-                    if new_value not in (None, "") and new_value != "********":
-                        continue
-                    if key in stored:
-                        new_config[key] = stored[key]
-                obj.config = new_config
-        super().save_model(request, obj, form, change)
-
-
-class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
-    list_display = ("id", "created_at", "created_by")
-    readonly_fields = ("id", "created_at", "modified_at")
-    show_search_mode_selector = False
-    change_form_template = "admin/archivebox_change_form.html"
-
-    def get_admin_toolbar_actions(self, request, obj):
-        """Return extra action button dicts for the shared change-form toolbar.
-
-        See ``templates/admin/includes/archivebox_toolbar.html`` for the
-        accepted keys. Default: no extras (toolbar renders Save/History/Delete
-        plus any ``django-object-actions`` change_actions).
-        """
-        return []
-
-    def render_change_form(self, request, context, add=False, change=False, form_url="", obj=None):
-        context.setdefault("archivebox_admin_actions", self.get_admin_toolbar_actions(request, obj))
-        return super().render_change_form(request, context, add=add, change=change, form_url=form_url, obj=obj)
-
-    def get_default_search_mode(self) -> str:
-        # The shared changelist template always asks every admin for a default
-        # search mode, even when the search-mode toggle is hidden.
-        return "meta"
-
-    def get_form(
-        self,
-        request: HttpRequest,
-        obj: models.Model | None = None,
-        change: bool = False,
-        **kwargs: object,
-    ):
-        form = super().get_form(request, obj, change=change, **kwargs)
-        if "created_by" in form.base_fields:
-            form.base_fields["created_by"].initial = request.user
-        return form
-
-    def get_urls(self):
-        """Swap the per-object admin URLs from ``<path:object_id>`` to
-        ``<hexuuid:object_id>`` so canonical change/delete/history URLs use the
-        32-char hex form. The hyphenated form still resolves because the
-        converter's regex accepts both — Django reverses through ``to_url``
-        which always emits hex, so links in templates / changelists / inline
-        formsets all canonicalize automatically.
-
-        Non-UUID PKs (an ``IntegerField`` PK on some legacy table, for example)
-        won't match the converter's regex and fall back to the default
-        ``<path:object_id>`` patterns we still include after our swap.
-        """
-        info = self.opts.app_label, self.opts.model_name
-        object_routes = [
-            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="{}_{}_history".format(*info)),
-            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="{}_{}_delete".format(*info)),
-            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="{}_{}_change".format(*info)),
-        ]
-        # Append after super().get_urls() so our patterns are the
-        # *last-registered* ones with the canonical admin URL names — Django's
-        # reverse() picks the later registration when names collide, which is
-        # how we make ``reverse("admin:app_model_change", args=[obj.pk])``
-        # emit the hex form. The original ``<path:object_id>`` routes stay in
-        # place as a fallback for non-UUID PKs and for resolving inbound
-        # hyphenated URLs (the ``hexuuid`` regex accepts both forms anyway).
-        return super().get_urls() + object_routes

archivebox/base_models/models.py

@@ -1,430 +0,0 @@
-"""Base models using UUIDv7 for all id fields."""
-
-__package__ = "archivebox.base_models"
-
-import json
-import shutil
-from typing import Any
-
-from archivebox.uuid_compat import CompactUUIDField, uuid7
-from pathlib import Path
-
-from django.core.exceptions import FieldDoesNotExist
-from django.db import models
-from django.db.models import F
-from django.db import transaction
-from django.db.models.signals import pre_delete
-from django.utils import timezone
-from django.contrib.auth import get_user_model
-from django.urls import reverse_lazy
-from django.conf import settings
-
-from django_stubs_ext.db.models import TypedModelMeta
-
-from archivebox.config import CONSTANTS
-
-
-class PagedQuerySet(models.QuerySet):
-    def paged_iterator(self, chunk_size: int = 500):
-        """
-        Iterate rows using bounded keyset pages instead of one streaming cursor.
-
-        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
-        cursor open until the full queryset is exhausted. That is fine for
-        read-only exports, but update/migration code does filesystem work and
-        writes while iterating; a long-lived read cursor there can stretch lock
-        waits across thousands of rows. This respects the queryset's existing
-        filters, order_by(), select_related(), and prefetch_related() state; if
-        no ordering is defined, it falls back to primary-key order.
-        """
-        pk_field = self.model._meta.pk.name
-        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
-
-        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        ordering = []
-        for term in raw_ordering:
-            descending = term.startswith("-")
-            field_name = term[1:] if descending else term
-            if field_name == "pk":
-                field_name = pk_field
-            ordering.append(f"-{field_name}" if descending else field_name)
-
-        ordered_field_names = [term.removeprefix("-") for term in ordering]
-        try:
-            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
-                offset = 0
-                while True:
-                    batch = list(self[offset : offset + chunk_size])
-                    if not batch:
-                        break
-                    yield from batch
-                    offset += chunk_size
-                return
-        except (AttributeError, FieldDoesNotExist):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
-        if not any(field_name in unique_field_names for field_name in ordered_field_names):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        last_values = None
-        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
-        while True:
-            batch_qs = self.order_by(*ordering)
-            if last_values is not None:
-                page_filter = models.Q()
-                for idx, term in enumerate(ordering):
-                    descending = term.startswith("-")
-                    field_name = term[1:] if descending else term
-                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
-                    comparison = "lt" if descending else "gt"
-                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
-                # Anchor the OR tie-breaks with an explicit leading range so
-                # SQLite/PG can seek the existing order index instead of
-                # rescanning its prefix for every keyset page.
-                first_comparison = "lte" if ordering[0].startswith("-") else "gte"
-                batch_qs = batch_qs.filter(
-                    page_filter,
-                    **{f"{ordered_field_names[0]}__{first_comparison}": last_values[0]},
-                )
-
-            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
-            if not batch_rows:
-                break
-
-            pk_idx = value_field_names.index(pk_field)
-            row_ids = [row[pk_idx] for row in batch_rows]
-            rows_by_id = {obj.pk: obj for obj in self.filter(pk__in=row_ids).order_by()}
-
-            for row in batch_rows:
-                row_id = row[pk_idx]
-                obj = rows_by_id.get(row_id)
-                if obj is not None:
-                    yield obj
-
-            last_values = batch_rows[-1][: len(ordered_field_names)]
-
-
-def normalize_config_json_values(config: Any) -> Any:
-    if not isinstance(config, dict):
-        return config
-
-    normalized = dict(config)
-    for key, value in list(normalized.items()):
-        if not isinstance(value, str) or len(value) < 2:
-            continue
-        if value[:1] != '"' or value[-1:] != '"':
-            continue
-        try:
-            decoded = json.loads(value)
-        except ValueError:
-            continue
-        if isinstance(decoded, str):
-            normalized[key] = decoded
-    return normalized
-
-
-def get_or_create_system_user_pk(username="system"):
-    User = get_user_model()
-    # If there's exactly one superuser, use that for all system operations
-    if User.objects.filter(is_superuser=True).count() == 1:
-        return User.objects.filter(is_superuser=True).values_list("pk", flat=True)[0]
-    # Otherwise get or create the system user
-    user, _ = User.objects.get_or_create(
-        username=username,
-        defaults={"is_staff": True, "is_superuser": True, "email": "", "password": "!"},
-    )
-    return user.pk
-
-
-class AutoDateTimeField(models.DateTimeField):
-    """DateTimeField that automatically updates on save (legacy compatibility)."""
-
-    def pre_save(self, model_instance, add):
-        if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
-            value = timezone.now()
-            setattr(model_instance, self.attname, value)
-            return value
-        return super().pre_save(model_instance, add)
-
-
-class ModelWithUUID(models.Model):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
-    created_at = models.DateTimeField(default=timezone.now, db_index=True)
-    modified_at = models.DateTimeField(auto_now=True)
-    created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL,
-        on_delete=models.CASCADE,
-        default=get_or_create_system_user_pk,
-        null=False,
-        db_index=True,
-    )
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def __str__(self) -> str:
-        return f"[{self.id}] {self.__class__.__name__}"
-
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
-    @property
-    def api_url(self) -> str:
-        return str(reverse_lazy("api-1:get_any", args=[self.id]))
-
-    @property
-    def api_docs_url(self) -> str:
-        return f"/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}"
-
-
-class ModelWithNotes(models.Model):
-    """Mixin for models with a notes field."""
-
-    notes = models.TextField(blank=True, null=False, default="")
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-
-class ModelWithHealthStats(models.Model):
-    """Mixin for models with health tracking fields."""
-
-    num_uses_failed = models.PositiveIntegerField(default=0)
-    num_uses_succeeded = models.PositiveIntegerField(default=0)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
-    @property
-    def health(self) -> int:
-        total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
-        return round((self.num_uses_succeeded / total) * 100)
-
-    def increment_health_stats(self, success: bool):
-        """Atomically increment success or failure counter using F() expression."""
-        field = "num_uses_succeeded" if success else "num_uses_failed"
-        type(self).objects.filter(pk=self.pk).update(
-            **{
-                field: F(field) + 1,
-                "modified_at": timezone.now(),
-            },
-        )
-
-
-class ModelWithConfig(models.Model):
-    """Mixin for models with a JSON config field."""
-
-    config = models.JSONField(default=dict, null=True, blank=True, editable=True)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def save(self, *args, **kwargs):
-        normalized_config = normalize_config_json_values(self.config)
-        if normalized_config != self.config:
-            self.config = normalized_config
-            update_fields = kwargs.get("update_fields")
-            if update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
-        super().save(*args, **kwargs)
-
-
-class ModelWithDeleteAfter(models.Model):
-    delete_after_final_statuses: tuple[str, ...] = ()
-    delete_at = models.DateTimeField(default=None, null=True, blank=True, db_index=True)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def save(self, *args, **kwargs):
-        update_fields = kwargs.get("update_fields")
-        if self.delete_at is None:
-            self.set_delete_at_from_config()
-            if self.delete_at is not None and update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
-        super().save(*args, **kwargs)
-
-    def get_delete_after_config_value(self):
-        from archivebox.config.common import get_config
-
-        return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
-
-    def set_delete_at_from_config(self, config_value=None) -> bool:
-        if self.delete_at is not None:
-            return False
-
-        from archivebox.config.common import parse_delete_after
-
-        duration = parse_delete_after(self.get_delete_after_config_value() if config_value is None else config_value)
-        if duration is None:
-            return False
-
-        self.delete_at = (self.created_at or timezone.now()) + duration
-        return True
-
-    @classmethod
-    def missing_delete_at_candidates(cls):
-        return cls.objects.none()
-
-    @classmethod
-    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True, backfill_cursors: dict | None = None) -> int:
-        if backfill_missing:
-            # Bound the scanned rows BEFORE evaluating JSON policy/parent
-            # joins. LIMIT on the policy query only bounded its output: finding
-            # no matches could scan the entire history with a SQLite read lock.
-            # The runner keeps a keyset cursor between idle passes, so disabled
-            # policies cannot starve older rows. Restarting safely repeats work.
-            page = cls.objects.order_by("-pk")
-            before = backfill_cursors.get(cls) if backfill_cursors is not None else None
-            if before is not None:
-                page = page.filter(pk__lt=before)
-            page_ids = list(page.values_list("pk", flat=True)[:batch_size])
-            if backfill_cursors is not None:
-                backfill_cursors[cls] = page_ids[-1] if len(page_ids) == batch_size else None
-            missing_delete_at = list(cls.missing_delete_at_candidates().filter(pk__in=page_ids).order_by())
-            for obj in missing_delete_at:
-                if obj.set_delete_at_from_config():
-                    cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
-                        delete_at=obj.delete_at,
-                        modified_at=timezone.now(),
-                    )
-
-        # Keep the expiration sweep anchored on delete_at. Some large tables
-        # have millions of final-status rows but almost no retained rows; adding
-        # the status filter before SQLite has narrowed by delete_at can make the
-        # runner scan the hot status index before it claims new work.
-        due_pks = list(
-            cls.objects.filter(delete_at__isnull=False, delete_at__lte=timezone.now())
-            .order_by("delete_at", "pk")
-            .values_list("pk", flat=True)[:batch_size],
-        )
-        if not due_pks:
-            return 0
-
-        queryset = cls.objects.filter(pk__in=due_pks)
-        if cls.delete_after_final_statuses:
-            queryset = queryset.filter(status__in=cls.delete_after_final_statuses)
-
-        count = 0
-        expired = list(queryset.order_by("delete_at", "pk"))
-        for obj in expired:
-            obj.delete()
-            count += 1
-        return count
-
-
-class ModelWithOutputDir(ModelWithUUID):
-    class Meta(ModelWithUUID.Meta):
-        abstract = True
-
-    _delete_signal_registered = False
-
-    # Saving metadata must not resolve or create output paths. The runner and
-    # explicit artifact writers create their own directories when writing files;
-    # on_commit would still make HTTP requests wait on remote archive storage.
-
-    @property
-    def output_dir_parent(self) -> str:
-        return f"{self._meta.model_name}s"
-
-    @property
-    def output_dir_name(self) -> str:
-        return str(self.id)
-
-    @property
-    def output_dir_str(self) -> str:
-        return f"{self.output_dir_parent}/{self.output_dir_name}"
-
-    @property
-    def output_dir(self) -> Path:
-        raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
-
-    def output_paths_for_delete(self) -> tuple[Path, ...]:
-        return (Path(self.output_dir),)
-
-    @classmethod
-    def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
-        data_dir = CONSTANTS.DATA_DIR.resolve()
-        safe_paths = []
-        for raw_path in paths:
-            path = Path(raw_path)
-            is_safe = False
-            for candidate in (path.absolute(), path.resolve()):
-                try:
-                    candidate.relative_to(data_dir)
-                    is_safe = True
-                    break
-                except ValueError:
-                    continue
-            if not is_safe:
-                raise ValueError(f"Refusing to delete output path outside DATA_DIR: {path}")
-            safe_paths.append(path)
-        return tuple(safe_paths)
-
-    @classmethod
-    def delete_output_paths(cls, paths) -> None:
-        for path in cls.validate_output_paths_for_delete(paths):
-            if path.is_symlink() or path.is_file():
-                path.unlink(missing_ok=True)
-            elif path.is_dir():
-                shutil.rmtree(path)
-            # lstat also catches dangling symlinks and propagates mount and
-            # permission errors rather than mistaking them for absence.
-            try:
-                path.lstat()
-            except FileNotFoundError:
-                continue
-            raise OSError(f"Output path still exists after deletion: {path}")
-
-    def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
-        """Capture output paths before DB deletion and remove them after commit."""
-        paths = self.validate_output_paths_for_delete(self.output_paths_for_delete())
-        transaction.on_commit(lambda: self.delete_output_paths(paths), using=using)
-
-    @classmethod
-    def register_delete_signal(cls) -> None:
-        if cls._delete_signal_registered:
-            return
-
-        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
-            if getattr(origin, "_output_files_deleted", False):
-                return
-            if not isinstance(instance, ModelWithOutputDir):
-                return
-            instance.schedule_delete_cleanup(using=using)
-
-        pre_delete.connect(
-            schedule_output_dir_cleanup,
-            dispatch_uid="archivebox.output_dir_cleanup_on_delete",
-            weak=False,
-        )
-        cls._delete_signal_registered = True

archivebox/cli/__init__.py

@@ -1,232 +1,155 @@
-__package__ = "archivebox.cli"
-__command__ = "archivebox"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox'
+
 import os
 import sys
-from contextlib import redirect_stdout
+import argparse
+
+from typing import Optional, Dict, List, IO, Union
+from pathlib import Path
+
+from ..config import OUTPUT_DIR, check_data_folder, check_migrations
+
 from importlib import import_module
 
-import rich_click as click
-from rich.console import Console
+CLI_DIR = Path(__file__).resolve().parent
 
-from archivebox.config.version import VERSION
+# these common commands will appear sorted before any others for ease-of-use
+meta_cmds = ('help', 'version')                               # dont require valid data folder at all
+main_cmds = ('init', 'config', 'setup')                       # dont require existing db present
+archive_cmds = ('add', 'remove', 'update', 'list', 'status')  # require existing db present
+fake_db = ("oneshot",)                                        # use fake in-memory db
 
-STDERR = Console(stderr=True)
+display_first = (*meta_cmds, *main_cmds, *archive_cmds)
+
+# every imported command module must have these properties in order to be valid
+required_attrs = ('__package__', '__command__', 'main')
+
+# basic checks to make sure imported files are valid subcommands
+is_cli_module = lambda fname: fname.startswith('archivebox_') and fname.endswith('.py')
+is_valid_cli_module = lambda module, subcommand: (
+    all(hasattr(module, attr) for attr in required_attrs)
+    and module.__command__.split(' ')[-1] == subcommand
+)
 
 
-if "--debug" in sys.argv:
-    os.environ["DEBUG"] = "True"
-    sys.argv.remove("--debug")
+def list_subcommands() -> Dict[str, str]:
+    """find and import all valid archivebox_<subcommand>.py files in CLI_DIR"""
 
-# Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
-# `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
-# the subcommand executes. `--quick-init` is the old server spelling. Strip either from
-# argv here so each subcommand's own click parser never sees it. Ignored for `help` and
-# `init` themselves.
-if "--init" in sys.argv or "--quick-init" in sys.argv:
-    sys.argv = [arg for arg in sys.argv if arg not in ("--init", "--quick-init")]
-    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+    COMMANDS = []
+    for filename in os.listdir(CLI_DIR):
+        if is_cli_module(filename):
+            subcommand = filename.replace('archivebox_', '').replace('.py', '')
+            module = import_module('.archivebox_{}'.format(subcommand), __package__)
+            assert is_valid_cli_module(module, subcommand)
+            COMMANDS.append((subcommand, module.main.__doc__))
+            globals()[subcommand] = module.main
+
+    display_order = lambda cmd: (
+        display_first.index(cmd[0])
+        if cmd[0] in display_first else
+        100 + len(cmd[0])
+    )
+
+    return dict(sorted(COMMANDS, key=display_order))
 
 
-class ArchiveBoxGroup(click.Group):
-    """lazy loading click group for archivebox commands"""
+def run_subcommand(subcommand: str,
+                   subcommand_args: List[str]=None,
+                   stdin: Optional[IO]=None,
+                   pwd: Union[Path, str, None]=None) -> None:
+    """Run a given ArchiveBox subcommand with the given list of args"""
 
-    meta_commands = {
-        "help": "archivebox.cli.archivebox_help.main",
-        "version": "archivebox.cli.archivebox_version.main",
-        "mcp": "archivebox.cli.archivebox_mcp.main",
-        "oneshot": "archivebox.cli.archivebox_oneshot.main",
-    }
-    setup_commands = {
-        "init": "archivebox.cli.archivebox_init.main",
-        "install": "archivebox.cli.archivebox_install.main",
-    }
-    # Model commands (CRUD operations via subcommands)
-    model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot.main",
-        "archiveresult": "archivebox.cli.archivebox_archiveresult.main",
-        "tag": "archivebox.cli.archivebox_tag.main",
-        "binary": "archivebox.cli.archivebox_binary.main",
-        "process": "archivebox.cli.archivebox_process.main",
-        "machine": "archivebox.cli.archivebox_machine.main",
-        "persona": "archivebox.cli.archivebox_persona.main",
-    }
-    archive_commands = {
-        # High-level commands
-        "add": "archivebox.cli.archivebox_add.main",
-        "extract": "archivebox.cli.archivebox_extract.main",
-        "list": "archivebox.cli.archivebox_list.main",
-        "remove": "archivebox.cli.archivebox_remove.main",
-        "run": "archivebox.cli.archivebox_run.main",
-        "update": "archivebox.cli.archivebox_update.main",
-        "status": "archivebox.cli.archivebox_status.main",
-        "search": "archivebox.cli.archivebox_search.main",
-        "config": "archivebox.cli.archivebox_config.main",
-        "schedule": "archivebox.cli.archivebox_schedule.main",
-        "server": "archivebox.cli.archivebox_server.main",
-        "shell": "archivebox.cli.archivebox_shell.main",
-        "manage": "archivebox.cli.archivebox_manage.main",
-        # Introspection commands
-        "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
-    }
-    all_subcommands = {
-        **meta_commands,
-        **setup_commands,
-        **model_commands,
-        **archive_commands,
-    }
-    renamed_commands = {
-        "setup": "install",
-        "import": "add",
-        "archive": "add",
-    }
+    subcommand_args = subcommand_args or []
 
-    @classmethod
-    def get_canonical_name(cls, cmd_name):
-        return cls.renamed_commands.get(cmd_name, cmd_name)
+    if subcommand not in meta_cmds:
+        from ..config import setup_django
 
-    @classmethod
-    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
-        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
-        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)
+        cmd_requires_db = subcommand in archive_cmds
+        init_pending = '--init' in subcommand_args or '--quick-init' in subcommand_args
 
-    @classmethod
-    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
-        if not cls._needs_django_for_lazy_import(cmd_name):
-            return
+        if cmd_requires_db:
+            check_data_folder(pwd)
 
-        from django.apps import apps
+        setup_django(in_memory_db=subcommand in fake_db, check_db=cmd_requires_db and not init_pending)
 
-        if apps.ready:
-            return
+        if cmd_requires_db:
+            check_migrations()
 
-        from archivebox.config.django import setup_django
-
-        setup_django()
-
-    def get_command(self, ctx, cmd_name):
-        # handle renamed commands
-        if cmd_name in self.renamed_commands:
-            new_name = self.renamed_commands[cmd_name]
-            STDERR.print(
-                f" [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`",
-            )
-            cmd_name = new_name
-            ctx.invoked_subcommand = cmd_name
-
-        # handle lazy loading of commands
-        if cmd_name in self.all_subcommands:
-            self._setup_django_for_lazy_import(cmd_name)
-            return self._lazy_load(cmd_name)
-
-        # fall-back to using click's default command lookup
-        return super().get_command(ctx, cmd_name)
-
-    @classmethod
-    def _lazy_load(cls, cmd_name_or_path):
-        import_path = cls.all_subcommands.get(cmd_name_or_path)
-        if import_path is None:
-            import_path = cmd_name_or_path
-        modname, funcname = import_path.rsplit(".", 1)
-
-        mod = import_module(modname)
-        func = vars(mod)[funcname]
-
-        if func.__doc__ is None:
-            raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
-
-        return func
+    module = import_module('.archivebox_{}'.format(subcommand), __package__)
+    module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore
 
 
-@click.group(cls=ArchiveBoxGroup, invoke_without_command=True)
-@click.option("--help", "-h", is_flag=True, help="Show help")
-@click.version_option(VERSION, "-v", "--version", package_name="archivebox", message="%(version)s")
-@click.pass_context
-def cli(ctx, help=False):
-    """ArchiveBox: The self-hosted internet archive"""
+SUBCOMMANDS = list_subcommands()
 
-    subcommand = ArchiveBoxGroup.get_canonical_name(ctx.invoked_subcommand)
-
-    # if --help is passed or no subcommand is given, show custom help message
-    if help or ctx.invoked_subcommand is None:
-        ctx.invoke(ctx.command.get_command(ctx, "help"))
-
-    # if the subcommand is in archive_commands or model_commands,
-    # then we need to set up the django environment and check that we're in a valid data folder
-    wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
-    if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
-        try:
-            # RUNSERVER_WORKER sets the web-worker markers on its own child.
-            # Setting them on the server parent leaks them through supervisord
-            # into runners waiting for takeover, which then register themselves
-            # as web workers and trigger spurious runner_watch reloads.
-            from archivebox.config.django import setup_django
-            from archivebox.misc.checks import check_data_folder, check_migrations
-
-            setup_django()
-            if os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1" and subcommand not in ("init", "help"):
-                # Universal `--init` was passed: build/upgrade the data folder before
-                # the regular preflight runs, so it succeeds on a fresh dir and an
-                # out-of-date schema both. Drop the env var afterwards so spawned
-                # subprocesses (supervisord workers, daphne, runner, etc.) inherit
-                # a clean env and don't re-trigger init in every child.
-                from archivebox.cli.archivebox_init import init as archivebox_init
-
-                archivebox_init(quick=True)
-                os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
-            check_data_folder()
-            if subcommand != "update" and not check_migrations(auto_apply=True):
-                from archivebox.machine.models import Machine
-
-                # Host registration and config reconciliation belong to CLI
-                # startup; get_config() is also used by read-only HTTP views.
-                Machine.current()
-        except Exception as e:
-            STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
-            if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
-                raise
+class NotProvided:
+    pass
 
 
-def main(args=None, prog_name=None):
-    from archivebox.config.constants import CONSTANTS
+def main(args: Optional[List[str]]=NotProvided, stdin: Optional[IO]=NotProvided, pwd: Optional[str]=None) -> None:
+    args = sys.argv[1:] if args is NotProvided else args
+    stdin = sys.stdin if stdin is NotProvided else stdin
 
-    os.environ.setdefault("ABX_PLUGINS_DIR", str(CONSTANTS.USER_PLUGINS_DIR))
+    subcommands = list_subcommands()
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description='ArchiveBox: The self-hosted internet archive',
+        add_help=False,
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--help', '-h',
+        action='store_true',
+        help=subcommands['help'],
+    )
+    group.add_argument(
+        '--version',
+        action='store_true',
+        help=subcommands['version'],
+    )
+    group.add_argument(
+        "subcommand",
+        type=str,
+        help= "The name of the subcommand to run",
+        nargs='?',
+        choices=subcommands.keys(),
+        default=None,
+    )
+    parser.add_argument(
+        "subcommand_args",
+        help="Arguments for the subcommand",
+        nargs=argparse.REMAINDER,
+    )
+    command = parser.parse_args(args or ())
 
-    # show `docker run archivebox xyz` in help messages if running in docker
-    IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
-    IS_TTY = sys.stdin.isatty()
-    prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
+    if command.version:
+        command.subcommand = 'version'
+    elif command.help or command.subcommand is None:
+        command.subcommand = 'help'
 
-    previous_unraisablehook = sys.unraisablehook
+    if command.subcommand not in ('help', 'version', 'status'):
+        from ..logging_util import log_cli_command
 
-    def ignore_shutdown_unraisable(unraisable):
-        if isinstance(unraisable.exc_value, (KeyboardInterrupt, SystemExit)):
-            return
-        previous_unraisablehook(unraisable)
+        log_cli_command(
+            subcommand=command.subcommand,
+            subcommand_args=command.subcommand_args,
+            stdin=stdin,
+            pwd=pwd or OUTPUT_DIR
+        )
 
-    sys.unraisablehook = ignore_shutdown_unraisable
-    try:
-        cli(args=args, prog_name=prog_name, standalone_mode=False)
-    except click.Abort:
-        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
-        raise SystemExit(130) from None
-    except click.ClickException as err:
-        err.show()
-        raise SystemExit(err.exit_code) from None
-    except click.exceptions.Exit as err:
-        raise SystemExit(err.exit_code) from None
-    except KeyboardInterrupt:
-        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
-        raise SystemExit(130) from None
-    finally:
-        sys.unraisablehook = previous_unraisablehook
-        supervisor_util = sys.modules.get("archivebox.workers.supervisord_util")
-        if supervisor_util is not None:
-            # Short commands can start a foreground-owned search daemon too.
-            # Finish its cleanup before another CLI can borrow a dying stack.
-            with redirect_stdout(sys.stderr):
-                supervisor_util.stop_own_supervisord_process()
+    run_subcommand(
+        subcommand=command.subcommand,
+        subcommand_args=command.subcommand_args,
+        stdin=stdin,
+        pwd=pwd or OUTPUT_DIR,
+    )
+
+
+__all__ = (
+    'SUBCOMMANDS',
+    'list_subcommands',
+    'run_subcommand',
+    *SUBCOMMANDS.keys(),
+)
 
 
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_add.py

@@ -1,446 +1,140 @@
 #!/usr/bin/env python3
-from __future__ import annotations
 
-__package__ = "archivebox.cli"
-__command__ = "archivebox add"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox add'
 
 import sys
-import os
-import signal
-from pathlib import Path
+import argparse
 
-from typing import Any, TYPE_CHECKING
+from typing import List, Optional, IO
 
-import rich_click as click
-
-from archivebox.config import CONSTANTS
-from archivebox.misc.util import enforce_types, docstring
-from archivebox.misc.util import parse_filesize_to_bytes
+from ..main import add
+from ..util import docstring
+from ..parsers import PARSERS
+from ..config import OUTPUT_DIR, ONLY_NEW
+from ..logging_util import SmartFormatter, accept_stdin, stderr
 
 
-if TYPE_CHECKING:
-    from django.db.models import QuerySet
-    from archivebox.crawls.models import Crawl
-    from archivebox.core.models import Snapshot
-
-
-def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
-    from archivebox.misc.jsonl import read_args_or_stdin
-    from archivebox.misc.util import validate_url
-
-    urls: list[str] = []
-    for record in read_args_or_stdin(args):
-        url = record.get("url")
-        if isinstance(url, str) and url:
-            try:
-                urls.append(validate_url(url))
-            except ValueError as err:
-                raise click.BadParameter(str(err), param_hint="URL") from err
-
-        urls_field = record.get("urls")
-        if isinstance(urls_field, str):
-            for line in urls_field.splitlines():
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    try:
-                        urls.append(validate_url(line))
-                    except ValueError as err:
-                        raise click.BadParameter(str(err), param_hint="URL") from err
-
-    return urls
-
-
-@enforce_types
-def add(
-    urls: str | list[str],
-    snapshot_ids: list[str] | None = None,
-    depth: int | str = 0,
-    max_urls: int = 0,
-    crawl_max_size: int | str = 0,
-    crawl_timeout: int = 0,
-    snapshot_max_size: int | str = 0,
-    crawl_max_concurrent_snapshots: int | None = None,
-    tag: str = "",
-    url_allowlist: str = "",
-    url_denylist: str = "",
-    parser: str = "auto",
-    plugins: str = "",
-    persona: str = "Default",
-    index_only: bool = False,
-    bg: bool = False,
-    created_by_id: int | None = None,
-    config: dict[str, Any] | None = None,
-) -> tuple[Crawl, QuerySet[Snapshot]]:
-    """Add a URL list or imported URL document to a new Crawl."""
-
-    from rich import print
-
-    depth = int(depth)
-    max_urls = int(max_urls or 0)
-    crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
-    crawl_timeout = int(crawl_timeout or 0)
-    snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    from archivebox.config.permissions import USER, HOSTNAME
-    from archivebox.config.common import get_config
-
-    config_overrides = dict(config or {})
-    runtime_config = get_config()
-    crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
-    if crawl_max_concurrent_snapshots is None:
-        crawl_max_concurrent_snapshots = runtime_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
-    crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
-
-    if depth not in (0, 1, 2, 3, 4):
-        raise ValueError("Depth must be 0-4")
-    if max_urls < 0:
-        raise ValueError("max_urls must be >= 0")
-    if crawl_max_size < 0:
-        raise ValueError("crawl_max_size must be >= 0")
-    if crawl_timeout < 0:
-        raise ValueError("crawl_timeout must be >= 0")
-    if snapshot_max_size < 0:
-        raise ValueError("snapshot_max_size must be >= 0")
-    if crawl_max_concurrent_snapshots < 1:
-        raise ValueError("crawl_max_concurrent_snapshots must be >= 1")
-
-    # import models once django is set up
-    from archivebox.crawls.models import Crawl
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.personas.models import Persona
-    from archivebox.misc.logging_util import printable_filesize
-    from archivebox.misc.system import get_dir_size
-    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-    from django.utils import timezone
-
-    created_by_id = created_by_id or get_or_create_system_user_pk()
-    started_at = timezone.now()
-
-    if isinstance(urls, str):
-        source_text = urls
-    else:
-        from archivebox.misc.util import validate_url
-
-        source_text = "\n".join(validate_url(str(url)) for url in urls)
-
-    # 2. Create a new Crawl with inline URLs
-    # Foreground add must claim runner ownership before publishing runnable
-    # work. Otherwise an existing server runner can lease the new Crawl in the
-    # gap between Crawl.objects.create() and current_command(), then be killed
-    # by the newer add process halfway through its first Snapshot hook.
-    command = None
-    if not bg and not index_only:
-        from archivebox.machine.models import Process
-        from archivebox.core.takeover_util import current_command
-
-        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR)
-
-    cli_args = [*sys.argv]
-    if cli_args[0].lower().endswith("archivebox"):
-        cli_args[0] = "archivebox"
-    cmd_str = " ".join(cli_args)
-
-    timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
-
-    persona_name = (persona or "Default").strip() or "Default"
-    plugins = plugins or ""
-    persona_obj = Persona.get_or_create_named(persona_name)
-    effective_persona_config = get_config(persona=persona_obj)
-
-    crawl_config = {
-        "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
-        **({"INDEX_ONLY": True} if index_only else {}),
-        **({"PLUGINS": plugins} if plugins else {}),
-        **(
-            {"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots}
-            if crawl_max_concurrent_snapshots_override
-            and crawl_max_concurrent_snapshots != int(effective_persona_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS)
-            else {}
-        ),
-        **({"CRAWL_MAX_URLS": max_urls} if max_urls else {}),
-        **({"CRAWL_MAX_SIZE": crawl_max_size} if crawl_max_size else {}),
-        **({"CRAWL_TIMEOUT": crawl_timeout} if crawl_timeout else {}),
-        **({"SNAPSHOT_MAX_SIZE": snapshot_max_size} if snapshot_max_size else {}),
-        **({"PARSER": parser} if parser != "auto" else {}),
-        **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
-        **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
-    }
-    # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
-    # priority for crawl-frozen keys. Runtime-derived execution keys are
-    # stripped by Crawl.save() and rederived when hooks run.
-    crawl_config.update(config_overrides)
-
-    # Crawl.urls is the user's submitted source, not a derived work queue for
-    # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
-    # callers audit or resume imports without losing RSS/Netscape/JSON metadata
-    # that is not representable as one plain URL per line.
-    try:
-        crawl = Crawl.objects.create(
-            urls=source_text,
-            max_depth=depth,
-            tags_str=tag,
-            persona_id=persona_obj.id,
-            label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
-            created_by_id=created_by_id,
-            status=Crawl.StatusChoices.QUEUED,
-            retry_at=None if index_only else timezone.now(),
-            config=crawl_config,
-        )
-        first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
-    except BaseException:
-        if command is not None:
-            command.mark_exited(exit_code=1)
-        raise
-    if command is not None:
-        command.url = first_url
-        command.save(update_fields=["url", "modified_at"])
-
-    print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
-    print(f"    [dim]First URL: {first_url}[/dim]")
-
-    # The runner parses Crawl.urls after claiming the Crawl, then persists the
-    # resulting depth-0 Snapshot facts. Recursive discoveries start at depth 1.
-
-    if index_only:
-        print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
-        return crawl, crawl.snapshot_set.none()
-
-    # 5. Start the crawl runner to process the queue
-    #    The runner will:
-    #    - Process Crawl -> create Snapshots from all URLs
-    #    - Process Snapshots -> run extractors
-    #    - Parser extractors discover new URLs -> create child Snapshots
-    #    - Repeat until max_depth reached
-
-    if bg:
-        # Background mode: just queue work and return (background runner via server will pick it up).
-        print(
-            "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
-        )
-    else:
-        # Foreground mode: run full crawl runner until all work is done
-        print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
-        from archivebox.core.takeover_util import command_owns_foreground_runner, standby_until_foreground_runner_needed
-        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
-
-        assert command is not None
-        exit_code = 0
-        interactive_interrupts = sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
-
-        def interrupt_hook() -> None:
-            raise KeyboardInterrupt
-
-        def crawl_is_complete() -> bool:
-            crawl.refresh_from_db(fields=["status"])
-            return crawl.status not in crawl.RUNNABLE_STATES
-
-        try:
-            try:
-                with (
-                    foreground_shutdown_signals(
-                        first_signal_message=None,
-                        interrupt_handlers={signal.SIGINT: interrupt_hook},
-                    ),
-                    foreground_parent_watchdog(),
-                ):
-                    while True:
-                        standby = standby_until_foreground_runner_needed(
-                            command,
-                            data_dir=CONSTANTS.DATA_DIR,
-                            work_is_complete=crawl_is_complete,
-                        )
-                        if standby["work_completed"]:
-                            break
-                        exit_code = run_runner_worker(
-                            ["--crawl-id", str(crawl.id)],
-                            name=f"worker_runner_add_{os.getpid()}",
-                            interactive_interrupts=interactive_interrupts,
-                            config=get_config(crawl=crawl),
-                        )
-                        crawl.refresh_from_db(fields=["status", "retry_at"])
-                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
-                            break
-                        # A shared supervisord can be stopped by another
-                        # foreground owner (for example `archivebox server`
-                        # shutting down) and stop the one-shot add runner while
-                        # this add's crawl is still runnable. Keep the add
-                        # process alive so it can re-enter the normal
-                        # foreground-runner ownership loop and finish its crawl.
-                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
-                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
-                            continue
-                        if exit_code == 0:
-                            break
-                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
-                            continue
-                        raise SystemExit(exit_code)
-            except KeyboardInterrupt:
-                exit_code = 130
-                print("\n[red][X] archivebox add interrupted.[/red]")
-                print("[yellow]Hint: resume this crawl with:[/yellow]")
-                print(f"    [green]archivebox run --crawl-id={crawl.id}[/green]")
-                raise SystemExit(exit_code)
-        finally:
-            command.mark_exited(exit_code=exit_code)
-            stop_own_supervisord_process()
-
-        # Print summary for foreground runs
-        try:
-            crawl.refresh_from_db()
-            try:
-                from django.db.models import Count, Sum
-
-                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("output_size"))
-                snapshots_count = int(totals["snapshot_count"] or 0)
-                total_bytes = int(totals["total_bytes"] or 0)
-            except Exception:
-                snapshots_count = crawl.snapshot_set.count()
-                total_bytes, _, _ = get_dir_size(crawl.output_dir)
-            total_size = printable_filesize(total_bytes)
-            total_time = timezone.now() - started_at
-            total_seconds = int(total_time.total_seconds())
-            mins, secs = divmod(total_seconds, 60)
-            hours, mins = divmod(mins, 60)
-            if hours:
-                duration_str = f"{hours}h {mins}m {secs}s"
-            elif mins:
-                duration_str = f"{mins}m {secs}s"
-            else:
-                duration_str = f"{secs}s"
-
-            # Output dir relative to DATA_DIR
-            try:
-                rel_output = Path(crawl.output_dir).relative_to(CONSTANTS.DATA_DIR)
-                rel_output_str = f"./{rel_output}"
-            except Exception:
-                rel_output_str = str(crawl.output_dir)
-
-            from archivebox.core.routes_util import build_admin_url
-
-            admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
-
-            if snapshots_count:
-                print("\n[bold]snapshot output saved to:[/bold]")
-                for snapshot in crawl.snapshot_set.order_by("created_at").iterator():
-                    click.echo(f"  {snapshot.output_dir}")
-            print("\n[bold]crawl logs and setup:[/bold]")
-            click.echo(f"  {rel_output_str}")
-            click.echo(f"  {admin_url}")
-            print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
-            print(f"[bold]total size:[/bold] {total_size}")
-            print(f"[bold]total time:[/bold] {duration_str}")
-        except Exception:
-            # Summary is best-effort; avoid failing the command if something goes wrong
-            pass
-
-    # 6. Return the list of Snapshots in this crawl
-    snapshots = crawl.snapshot_set.all()
-    return crawl, snapshots
-
-
-@click.command()
-@click.option(
-    "--depth",
-    "-d",
-    type=click.Choice([str(i) for i in range(5)]),
-    default="0",
-    help="Recursively archive linked pages up to N hops away",
-)
-@click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
-@click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
-@click.option("--crawl-timeout", type=int, default=0, help="Maximum total crawl runtime in seconds (0 = unlimited)")
-@click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)")
-@click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl")
-@click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
-@click.option("--url-allowlist", "--domain-allowlist", default="", help="Comma-separated URL/domain allowlist for this crawl")
-@click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
-@click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
-@click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
-@click.option("--extract", default="", help="Comma-separated plugins or output types to extract e.g. title,pdf,text/html,image,...")
-@click.option("--persona", default="Default", help="Authentication profile to use when archiving")
-@click.option(
-    "--only-new/--no-only-new",
-    "only_new",
-    default=None,
-    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
-    "Pass --no-only-new to force re-archive of URLs that already exist.",
-)
-@click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
-@click.option("--overwrite", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
-@click.option("--update", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
-@click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
-@click.argument("urls", nargs=-1, type=click.Path())
 @docstring(add.__doc__)
-def main(**kwargs):
-    """Add a new URL or list of URLs to your archive"""
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=add.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    parser.add_argument(
+        '--tag', '-t',
+        type=str,
+        default='',
+        help="Tag the added URLs with the provided tags e.g. --tag=tag1,tag2,tag3",
+    )
+    parser.add_argument(
+        '--update', #'-u',
+        action='store_true',
+        default=not ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
+        help="Also retry previously skipped/failed links when adding new links",
+    )
+    parser.add_argument(
+        '--update-all', #'-n',
+        action='store_true',
+        default=False, 
+        help="Also update ALL links in index when finished adding new links",
+    )
+    parser.add_argument(
+        '--index-only', #'-o',
+        action='store_true',
+        help="Add the links to the main index without archiving them",
+    )
+    parser.add_argument(
+        'urls',
+        nargs='*',
+        type=str,
+        default=None,
+        help=(
+            'URLs or paths to archive e.g.:\n'
+            '    https://getpocket.com/users/USERNAME/feed/all\n'
+            '    https://example.com/some/rss/feed.xml\n'
+            '    https://example.com\n'
+            '    ~/Downloads/firefox_bookmarks_export.html\n'
+            '    ~/Desktop/sites_list.csv\n'
+        )
+    )
+    parser.add_argument(
+        "--depth",
+        action="store",
+        default=0,
+        choices=[0, 1],
+        type=int,
+        help="Recursively archive all linked pages up to this many hops away"
+    )
+    parser.add_argument(
+        "--overwrite",
+        default=False,
+        action="store_true",
+        help="Re-archive URLs from scratch, overwriting any existing files"
+    )
+    parser.add_argument(
+        "--init", #'-i',
+        action='store_true',
+        help="Init/upgrade the curent data directory before adding",
+    )
+    parser.add_argument(
+        "--extract",
+        type=str,
+        help="Pass a list of the extractors to be used. If the method name is not correct, it will be ignored. \
+              This does not take precedence over the configuration",
+        default=""
+    )
+    parser.add_argument(
+        "--parser",
+        type=str,
+        help="Parser used to read inputted URLs.",
+        default="auto",
+        choices=["auto", *PARSERS.keys()],
+    )
+    command = parser.parse_args(args or ())
+    urls = command.urls
 
-    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    stdin_urls = ''
+    if not urls:
+        stdin_urls = accept_stdin(stdin)
 
-    with foreground_shutdown_signals(), foreground_parent_watchdog():
-        raw_urls = kwargs.pop("urls")
-        if raw_urls:
-            urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
-        elif not sys.stdin.isatty():
-            urls = sys.stdin.read()
-        else:
-            urls = []
-        if not urls:
-            raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
-        if int(kwargs.get("max_urls") or 0) < 0:
-            raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
-        if int(kwargs.get("crawl_timeout") or 0) < 0:
-            raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout")
-        try:
-            kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
-        except ValueError as err:
-            raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
-        try:
-            kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
-        except ValueError as err:
-            raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
-        if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
-            raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
-
-        # Translate --only-new/--no-only-new into a crawl config override.
-        # add() takes config overrides as a dict; no per-flag kwargs.
-        overwrite = kwargs.pop("overwrite", False)
-        update = kwargs.pop("update", False)
-        only_new = kwargs.pop("only_new", None)
-        extract = kwargs.pop("extract", "")
-        if overwrite or update:
-            only_new = False
-        if only_new is not None:
-            kwargs["config"] = {"ONLY_NEW": bool(only_new)}
-        if extract:
-            from archivebox.plugins.discovery import get_plugin_catalog
-
-            catalog = get_plugin_catalog()
-            tokens = [token.strip() for token in extract.split(",") if token.strip()]
-            plugin_names = {name.lower(): name for name in catalog}
-            selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
-            selected += catalog.matching_output(tokens)
-            if not selected:
-                raise click.UsageError(f"No plugins found matching extract types: {extract}")
-            existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
-            kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
-
-        from archivebox.machine.models import Process
-
-        # CLI commands own their process record and logs. Queue model saves
-        # must not register a process implicitly: HTTP add() shares that path.
-        process = Process.current()
-        exit_code = 0
-        try:
-            add(urls=urls, **kwargs)
-        except BaseException:
-            exit_code = 1
-            raise
-        finally:
-            # --bg/--index-only return without the foreground runner's cleanup.
-            # Leaving them RUNNING makes the active-process index accumulate
-            # historical CLI commands forever, defeating bounded recovery.
-            if process.status == Process.StatusChoices.RUNNING:
-                process.mark_exited(exit_code=exit_code)
+    if (stdin_urls and urls) or (not stdin and not urls):
+        stderr(
+            '[X] You must pass URLs/paths to add via stdin or CLI arguments.\n',
+            color='red',
+        )
+        raise SystemExit(2)
+    add(
+        urls=stdin_urls or urls,
+        depth=command.depth,
+        tag=command.tag,
+        update=command.update,
+        update_all=command.update_all,
+        index_only=command.index_only,
+        overwrite=command.overwrite,
+        init=command.init,
+        extractors=command.extract,
+        parser=command.parser,
+        out_dir=pwd or OUTPUT_DIR,
+    )
 
 
-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
+
+
+# TODO: Implement these
+#
+# parser.add_argument(
+#     '--mirror', #'-m',
+#     action='store_true',
+#     help='Archive an entire site (finding all linked pages below it on the same domain)',
+# )
+# parser.add_argument(
+#     '--crawler', #'-r',
+#     choices=('depth_first', 'breadth_first'),
+#     help='Controls which crawler to use in order to find outlinks in a given page',
+#     default=None,
+# )

archivebox/cli/archivebox_archiveresult.py

@@ -1,383 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox archiveresult <action> [args...] [--filters]
-
-Manage ArchiveResult records (plugin extraction results).
-
-Actions:
-    create  - Emit plugin extraction request records for Snapshots
-    list    - List ArchiveResults as JSONL (with optional filters)
-    update  - Update ArchiveResults from stdin JSONL
-    delete  - Delete ArchiveResults from stdin JSONL
-
-Examples:
-    # Emit extraction requests; `archivebox run` schedules their parent snapshots
-    archivebox snapshot list --status=queued | archivebox archiveresult create
-    archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
-
-    # List with filters
-    archivebox archiveresult list --status=failed
-    archivebox archiveresult list --plugin=screenshot --status=succeeded
-
-    # Delete
-    archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
-
-    # Re-run failed extractions
-    archivebox archiveresult list --status=failed | archivebox run
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox archiveresult"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_util import apply_filters
-
-
-def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
-    return {
-        "type": "ArchiveResult",
-        "snapshot_id": str(snapshot_id),
-        "plugin": plugin,
-        "hook_name": hook_name,
-        "status": status,
-    }
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_archiveresults(
-    snapshot_id: str | None = None,
-    plugin: str | None = None,
-    status: str = "queued",
-) -> int:
-    """
-    Create ArchiveResult request records for Snapshots.
-
-    Reads Snapshot records from stdin and emits ArchiveResult request JSONL.
-    Pass-through: Non-Snapshot/ArchiveResult records are output unchanged.
-    If --plugin is specified, only emits requests for that plugin.
-    Otherwise, emits requests for all enabled snapshot hooks.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.config.common import get_config
-    from archivebox.plugins.hooks import discover_hooks
-    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
-    from archivebox.core.models import Snapshot
-
-    is_tty = sys.stdout.isatty()
-
-    # If snapshot_id provided directly, use that
-    if snapshot_id:
-        try:
-            snapshots = [Snapshot.objects.get(id=snapshot_id)]
-            pass_through_records = []
-        except Snapshot.DoesNotExist:
-            rprint(f"[red]Snapshot not found: {snapshot_id}[/red]", file=sys.stderr)
-            return 1
-    else:
-        # Read from stdin
-        records = list(read_stdin())
-        if not records:
-            rprint("[yellow]No Snapshot records provided via stdin[/yellow]", file=sys.stderr)
-            return 1
-
-        # Separate snapshot records from pass-through records
-        snapshot_ids = []
-        pass_through_records = []
-
-        for record in records:
-            record_type = record.get("type", "")
-
-            if record_type == TYPE_SNAPSHOT:
-                # Pass through the Snapshot record itself
-                pass_through_records.append(record)
-                if record.get("id"):
-                    snapshot_ids.append(record["id"])
-
-            elif record_type == TYPE_ARCHIVERESULT:
-                # ArchiveResult records: pass through if they have an id
-                if record.get("id"):
-                    pass_through_records.append(record)
-                # If no id, we could create it, but for now just pass through
-                else:
-                    pass_through_records.append(record)
-
-            elif record_type:
-                # Other typed records (Crawl, Tag, etc): pass through
-                pass_through_records.append(record)
-
-            elif record.get("id"):
-                # Untyped record with id - assume it's a snapshot ID
-                snapshot_ids.append(record["id"])
-
-        # Output pass-through records first
-        if not is_tty:
-            for record in pass_through_records:
-                write_record(record)
-
-        if not snapshot_ids:
-            if pass_through_records:
-                rprint(f"[dim]Passed through {len(pass_through_records)} records, no new snapshots to process[/dim]", file=sys.stderr)
-                return 0
-            rprint("[yellow]No valid Snapshot IDs in input[/yellow]", file=sys.stderr)
-            return 1
-
-        snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids))
-
-    if not snapshots:
-        rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
-        return 0 if pass_through_records else 1
-
-    created_count = 0
-    for snapshot in snapshots:
-        config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
-        hooks = [
-            hook
-            for hook in discover_hooks("Snapshot", filter_disabled=not plugin, config=config)
-            if not plugin or hook.parent.name == plugin
-        ]
-        for hook_path in hooks:
-            hook_name = hook_path.stem
-            plugin_name = hook_path.parent.name
-            if not is_tty:
-                write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
-            created_count += 1
-
-    rprint(f"[green]Created {created_count} extraction request records[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_archiveresults(
-    status: str | None = None,
-    plugin: str | None = None,
-    snapshot_id: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List ArchiveResults as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import ArchiveResult
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = ArchiveResult.objects.all().order_by("-start_ts")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "plugin": plugin,
-        "snapshot_id": snapshot_id,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for result in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "succeeded": "green",
-                "failed": "red",
-                "skipped": "dim",
-                "noresults": "dim",
-                "backoff": "magenta",
-            }.get(result.status, "dim")
-            rprint(
-                f"[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
-            )
-        else:
-            write_record(result.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} archive results[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_archiveresults(
-    status: str | None = None,
-) -> int:
-    """
-    Update ArchiveResults from stdin JSONL.
-
-    Reads ArchiveResult records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.core.models import ArchiveResult
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        result_id = record.get("id")
-        if not result_id:
-            continue
-
-        try:
-            result = ArchiveResult.objects.get(id=result_id)
-
-            # Apply updates from CLI flags
-            if status:
-                result.status = status
-
-            result.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(result.to_json())
-
-        except ArchiveResult.DoesNotExist:
-            rprint(f"[yellow]ArchiveResult not found: {result_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} archive results[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete ArchiveResults from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.core.models import ArchiveResult
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    result_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not result_ids:
-        rprint("[yellow]No valid archive result IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    results = ArchiveResult.objects.filter(id__in=result_ids)
-    count = results.count()
-
-    if count == 0:
-        rprint("[yellow]No matching archive results found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} archive results (dry run)[/yellow]", file=sys.stderr)
-        for result in results[:10]:
-            rprint(f"  [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}", file=sys.stderr)
-        if count > 10:
-            rprint(f"  ... and {count - 10} more", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = results.delete()
-    rprint(f"[green]Deleted {deleted_count} archive results[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage ArchiveResult records (plugin extraction results)."""
-    pass
-
-
-@main.command("create")
-@click.option("--snapshot-id", help="Snapshot ID to create results for")
-@click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
-    """Emit Snapshot plugin extraction requests as JSONL."""
-    sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
-
-
-@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, succeeded, failed, skipped)")
-@click.option("--plugin", "-p", help="Filter by plugin name")
-@click.option("--snapshot-id", help="Filter by snapshot ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(
-    status: str | None,
-    plugin: str | None,
-    snapshot_id: str | None,
-    limit: int | None,
-):
-    """List ArchiveResults as JSONL."""
-    sys.exit(
-        list_archiveresults(
-            status=status,
-            plugin=plugin,
-            snapshot_id=snapshot_id,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--status", "-s", help="Set status")
-def update_cmd(status: str | None):
-    """Update ArchiveResults from stdin JSONL."""
-    sys.exit(update_archiveresults(status=status))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete ArchiveResults from stdin JSONL."""
-    sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_binary.py

@@ -1,319 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox binary <action> [args...] [--filters]
-
-Manage Binary records (detected executables like chrome, wget, etc.).
-
-Actions:
-    create  - Create/register a Binary
-    list    - List Binaries as JSONL (with optional filters)
-    update  - Update Binaries from stdin JSONL
-    delete  - Delete Binaries from stdin JSONL
-
-Examples:
-    # List all binaries
-    archivebox binary list
-
-    # List specific binary
-    archivebox binary list --name=chrome
-
-    # List binaries with specific version
-    archivebox binary list --version__icontains=120
-
-    # Delete old binary entries
-    archivebox binary list --name=chrome | archivebox binary delete --yes
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox binary"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_util import apply_filters
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_binary(
-    name: str,
-    abspath: str,
-    version: str = "",
-) -> int:
-    """
-    Create/register a Binary.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    if not name or not abspath:
-        rprint("[red]Both --name and --abspath are required[/red]", file=sys.stderr)
-        return 1
-
-    try:
-        from archivebox.machine.models import Machine
-
-        machine = Machine.current()
-        created = not Binary.objects.filter(
-            machine=machine,
-            name=name,
-            abspath=abspath,
-            version=version,
-        ).exists()
-
-        # Mirror the Binary model lifecycle used elsewhere in the system so CLI
-        # records are owned by the current machine and can be safely piped into
-        # `archivebox run` without creating invalid rows missing machine_id.
-        binary = Binary.from_json(
-            {
-                "name": name,
-                "abspath": abspath,
-                "version": version,
-                "binproviders": "env",
-                "binprovider": "env",
-            },
-        )
-        if binary is None:
-            raise ValueError("failed to create binary record")
-
-        if not is_tty:
-            write_record(binary.to_json())
-
-        if created:
-            rprint(f"[green]Created binary: {name} at {abspath}[/green]", file=sys.stderr)
-        else:
-            rprint(f"[dim]Binary already exists: {name} at {abspath}[/dim]", file=sys.stderr)
-
-        return 0
-
-    except Exception as e:
-        rprint(f"[red]Error creating binary: {e}[/red]", file=sys.stderr)
-        return 1
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_binaries(
-    name: str | None = None,
-    abspath__icontains: str | None = None,
-    version__icontains: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Binaries as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Binary.objects.all().order_by("name", "-modified_at", "-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "name": name,
-        "abspath__icontains": abspath__icontains,
-        "version__icontains": version__icontains,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for binary in queryset:
-        if is_tty:
-            rprint(f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}")
-        else:
-            write_record(binary.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} binaries[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_binaries(
-    version: str | None = None,
-    abspath: str | None = None,
-) -> int:
-    """
-    Update Binaries from stdin JSONL.
-
-    Reads Binary records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        binary_id = record.get("id")
-        if not binary_id:
-            continue
-
-        try:
-            binary = Binary.objects.get(id=binary_id)
-
-            # Apply updates from CLI flags
-            if version:
-                binary.version = version
-            if abspath:
-                binary.abspath = abspath
-
-            binary.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(binary.to_json())
-
-        except Binary.DoesNotExist:
-            rprint(f"[yellow]Binary not found: {binary_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} binaries[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_binaries(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Binaries from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.machine.models import Binary
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    binary_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not binary_ids:
-        rprint("[yellow]No valid binary IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    binaries = Binary.objects.filter(id__in=binary_ids)
-    count = binaries.count()
-
-    if count == 0:
-        rprint("[yellow]No matching binaries found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} binaries (dry run)[/yellow]", file=sys.stderr)
-        for binary in binaries:
-            rprint(f"  {binary.name} {binary.abspath}", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = binaries.delete()
-    rprint(f"[green]Deleted {deleted_count} binaries[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Binary records (detected executables)."""
-    pass
-
-
-@main.command("create")
-@click.option("--name", "-n", required=True, help="Binary name (e.g., chrome, wget)")
-@click.option("--abspath", "-p", required=True, help="Absolute path to binary")
-@click.option("--version", "-v", default="", help="Binary version")
-def create_cmd(name: str, abspath: str, version: str):
-    """Create/register a Binary."""
-    sys.exit(create_binary(name=name, abspath=abspath, version=version))
-
-
-@main.command("list")
-@click.option("--name", "-n", help="Filter by name")
-@click.option("--abspath__icontains", help="Filter by path contains")
-@click.option("--version__icontains", help="Filter by version contains")
-@click.option("--limit", type=int, help="Limit number of results")
-def list_cmd(
-    name: str | None,
-    abspath__icontains: str | None,
-    version__icontains: str | None,
-    limit: int | None,
-):
-    """List Binaries as JSONL."""
-    sys.exit(
-        list_binaries(
-            name=name,
-            abspath__icontains=abspath__icontains,
-            version__icontains=version__icontains,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--version", "-v", help="Set version")
-@click.option("--abspath", "-p", help="Set path")
-def update_cmd(version: str | None, abspath: str | None):
-    """Update Binaries from stdin JSONL."""
-    sys.exit(update_binaries(version=version, abspath=abspath))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Binaries from stdin JSONL."""
-    sys.exit(delete_binaries(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_config.py

@@ -1,199 +1,64 @@
 #!/usr/bin/env python3
 
-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox config'
 
 import sys
-import toml
-import rich_click as click
-from rich import print
-from pathlib import Path
+import argparse
 
-from archivebox.misc.util import docstring, enforce_types
-from archivebox.misc.toml_util import CustomTOMLEncoder
+from typing import Optional, List, IO
+
+from ..main import config
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, accept_stdin
 
 
-def _format_toml(config: dict) -> str:
-    return toml.dumps(config, encoder=CustomTOMLEncoder()).strip().replace("\n\n", "\n")
-
-
-@enforce_types
-def config(
-    *keys,
-    get: bool = False,
-    set: bool = False,
-    search: bool = False,
-    reset: bool = False,
-    **kwargs,
-) -> None:
-    """Get and set your ArchiveBox project configuration values"""
-
-    from archivebox.misc.checks import check_data_folder
-    from archivebox.misc.logging_util import printable_config
-    from abx_plugins.plugins.base.utils import resolve_alias
-    from archivebox.config.collection import write_config_file
-    from archivebox.config import CONSTANTS_CONFIG
-    from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
-    from archivebox.plugins.discovery import discover_plugin_configs
-
-    check_data_folder()
-
-    FLAT_CONFIG = get_config().as_dict()
-    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
-    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
-    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
-    readable_config = {**writable_config, **readonly_config}
-    CONFIGS = get_all_configs()
-    plugin_schemas = {
-        plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
-    }
-    core_config_aliases = {
-        alias.upper(): field_name
-        for field_name, field in ArchiveBoxConfig.model_fields.items()
-        for alias in (field_name, str(field.alias or ""))
-        if alias
-    }
-
-    config_options: list[str] = list(kwargs.pop("key=value", []) or keys or [f"{key}={val}" for key, val in kwargs.items()])
-    no_args = not (get or set or reset or config_options)
-
-    matching_config = {}
-    if search:
-        if config_options:
-            search_terms = [key.strip().lower() for key in config_options]
-
-            for existing_key, value in readable_config.items():
-                if any(term in existing_key.lower() or term in str(value).lower() for term in search_terms):
-                    matching_config[existing_key] = value
-
-            for alias, key in core_config_aliases.items():
-                if key in readable_config and any(term in alias.lower() for term in search_terms):
-                    matching_config[key] = readable_config[key]
-
-            for schema in plugin_schemas.values():
-                for key, metadata in schema.items():
-                    if key in readable_config and any(term in key.lower() or term in str(metadata).lower() for term in search_terms):
-                        matching_config[key] = readable_config[key]
-
-        print(printable_config(matching_config))
-        raise SystemExit(not matching_config)
-
-    elif get or no_args:
-        if config_options:
-            config_options = [
-                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
-            ]
-            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
-            failed_config = [key for key in config_options if key not in readable_config]
-            if failed_config:
-                print("\n[red][X] These options failed to get[/red]")
-                print("    {}".format("\n    ".join(config_options)))
-                raise SystemExit(1)
-        else:
-            matching_config = readable_config
-
-        # Display core config sections
-        for config_section in CONFIGS.values():
-            section_header = config_section.toml_section_header
-            if isinstance(section_header, str) and section_header:
-                print(f"[grey53]\\[{section_header}][/grey53]")
-            else:
-                print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
-
-            kv_in_section = {key: val for key, val in dict(config_section).items() if key in matching_config}
-            print(_format_toml(kv_in_section))
-            print("[grey53]################################################################[/grey53]")
-
-        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
-        if readonly_keys:
-            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
-            print(_format_toml(readonly_keys))
-            print("[grey53]################################################################[/grey53]")
-
-        plugin_keys = {}
-
-        # Collect all plugin config keys
-        for schema in plugin_schemas.values():
-            for key in schema.keys():
-                if key in matching_config and key in writable_config:
-                    plugin_keys[key] = matching_config[key]
-
-        # Display all plugin config in single [PLUGINS] section
-        if plugin_keys:
-            print("[grey53]\\[PLUGINS][/grey53]")
-            print(_format_toml(plugin_keys))
-            print("[grey53]################################################################[/grey53]")
-
-        raise SystemExit(not matching_config)
-
-    elif set:
-        new_config = {}
-        failed_options = []
-        for line in config_options:
-            if line.startswith("#") or not line.strip():
-                continue
-            if "=" not in line:
-                print("[red][X] Config KEY=VALUE must have an = sign in it[/red]")
-                print(f"    {line}")
-                raise SystemExit(2)
-
-            raw_key, val = line.split("=", 1)
-            raw_key = raw_key.upper().strip()
-            key = core_config_aliases.get(raw_key) or resolve_alias(raw_key, plugin_schemas)
-            if key != raw_key:
-                print(
-                    f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
-                )
-
-            if key in writable_config:
-                new_config[key] = val.strip()
-            else:
-                failed_options.append(line)
-
-        if new_config:
-            before = FLAT_CONFIG
-            matching_config = write_config_file(new_config)
-            after = get_config().as_dict()
-            print(printable_config(matching_config))
-
-            side_effect_changes = {}
-            for key, val in after.items():
-                if key in FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
-                    side_effect_changes[key] = after[key]
-
-            if side_effect_changes:
-                print(file=sys.stderr)
-                print("[yellow][i] Note: This change also affected these other options that depended on it:[/yellow]", file=sys.stderr)
-                print("    {}".format(printable_config(side_effect_changes, prefix="    ")), file=sys.stderr)
-
-        if failed_options:
-            print()
-            print("[red][X] These options failed to set (check for typos):[/red]")
-            print("    {}".format("\n    ".join(failed_options)))
-            raise SystemExit(1)
-
-    elif reset:
-        print("[red][X] This command is not implemented yet.[/red]")
-        print("    Please manually remove the relevant lines from your config file:")
-        raise SystemExit(2)
-
-    else:
-        print("[red][X] You must pass either --get or --set, or no arguments to get the whole config.[/red]")
-        print("    archivebox config")
-        print("    archivebox config --get SOME_KEY")
-        print("    archivebox config --set SOME_KEY=SOME_VALUE")
-        raise SystemExit(2)
-
-
-@click.command()
-@click.option("--search", is_flag=True, help="Search config KEYs, VALUEs, and ALIASES for the given term")
-@click.option("--get", is_flag=True, help="Get the value for the given config KEYs")
-@click.option("--set", is_flag=True, help="Set the given KEY=VALUE config values")
-@click.option("--reset", is_flag=True, help="Reset the given KEY config values to their defaults")
-@click.argument("KEY=VALUE", nargs=-1, type=str)
 @docstring(config.__doc__)
-def main(**kwargs) -> None:
-    config(**kwargs)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=config.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
+    )
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        '--get', #'-g',
+        action='store_true',
+        help="Get the value for the given config KEYs",
+    )
+    group.add_argument(
+        '--set', #'-s',
+        action='store_true',
+        help="Set the given KEY=VALUE config values",
+    )
+    group.add_argument(
+        '--reset', #'-s',
+        action='store_true',
+        help="Reset the given KEY config values to their defaults",
+    )
+    parser.add_argument(
+        'config_options',
+        nargs='*',
+        type=str,
+        help='KEY or KEY=VALUE formatted config values to get or set',
+    )
+    command = parser.parse_args(args or ())
+
+    config_options_str = ''
+    if not command.config_options:
+        config_options_str = accept_stdin(stdin)
+
+    config(
+        config_options_str=config_options_str,
+        config_options=command.config_options,
+        get=command.get,
+        set=command.set,
+        reset=command.reset,
+        out_dir=pwd or OUTPUT_DIR,
+    )
 
 
-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/cli/archivebox_crawl.py

@@ -1,406 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox crawl <action> [args...] [--filters]
-
-Manage Crawl records.
-
-Actions:
-    create  - Create Crawl jobs from URLs
-    list    - List Crawls as JSONL (with optional filters)
-    update  - Update Crawls from stdin JSONL
-    delete  - Delete Crawls from stdin JSONL
-
-Examples:
-    # Create
-    archivebox crawl create https://example.com https://foo.com --depth=1
-    archivebox crawl create --tag=news https://example.com
-
-    # List with filters
-    archivebox crawl list --status=queued
-    archivebox crawl list --urls__icontains=example.com
-
-    # Update
-    archivebox crawl list --status=started | archivebox crawl update --status=queued
-
-    # Delete
-    archivebox crawl list --urls__icontains=spam.com | archivebox crawl delete --yes
-
-    # Full pipeline
-    archivebox crawl create https://example.com | archivebox snapshot create | archivebox run
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-from collections.abc import Iterable
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_util import apply_filters
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_crawl(
-    urls: Iterable[str],
-    depth: int = 0,
-    tag: str = "",
-    status: str = "queued",
-    created_by_id: int | None = None,
-) -> int:
-    """
-    Create a Crawl job from URLs.
-
-    Takes URLs as args or stdin, creates one Crawl with all URLs, outputs JSONL.
-    Pass-through: Records that are not URLs are output unchanged (for piping).
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
-    from archivebox.misc.util import validate_url
-    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.crawls.models import Crawl
-
-    created_by_id = created_by_id or get_or_create_system_user_pk()
-    is_tty = sys.stdout.isatty()
-
-    # Collect all input records
-    records = list(read_args_or_stdin(urls))
-
-    if not records:
-        rprint("[yellow]No URLs provided. Pass URLs as arguments or via stdin.[/yellow]", file=sys.stderr)
-        return 1
-
-    # Separate pass-through records from URL records
-    url_list = []
-    pass_through_records = []
-
-    for record in records:
-        record_type = record.get("type", "")
-
-        # Pass-through: output records that aren't URL/Crawl types
-        if record_type and record_type != TYPE_CRAWL and not record.get("url") and not record.get("urls"):
-            pass_through_records.append(record)
-            continue
-
-        # Handle existing Crawl records (just pass through with id)
-        if record_type == TYPE_CRAWL and record.get("id"):
-            pass_through_records.append(record)
-            continue
-
-        # Collect URLs
-        url = record.get("url")
-        if url:
-            try:
-                url_list.append(validate_url(str(url)))
-            except ValueError as err:
-                rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
-                return 1
-
-        # Handle 'urls' field (newline-separated)
-        urls_field = record.get("urls")
-        if urls_field:
-            for line in urls_field.split("\n"):
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    try:
-                        url_list.append(validate_url(line))
-                    except ValueError as err:
-                        rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
-                        return 1
-
-    # Output pass-through records first
-    if not is_tty:
-        for record in pass_through_records:
-            write_record(record)
-
-    if not url_list:
-        if pass_through_records:
-            # If we had pass-through records but no URLs, that's OK
-            rprint(f"[dim]Passed through {len(pass_through_records)} records, no new URLs[/dim]", file=sys.stderr)
-            return 0
-        rprint("[red]No valid URLs found[/red]", file=sys.stderr)
-        return 1
-
-    try:
-        # Build crawl record with all URLs as newline-separated string
-        crawl_record = {
-            "urls": "\n".join(url_list),
-            "max_depth": depth,
-            "tags_str": tag,
-            "status": status,
-            "label": "",
-        }
-
-        crawl = Crawl.from_json(crawl_record, overrides={"created_by_id": created_by_id})
-        if not crawl:
-            rprint("[red]Failed to create crawl[/red]", file=sys.stderr)
-            return 1
-
-        # Output JSONL record (only when piped)
-        if not is_tty:
-            write_record(crawl.to_json())
-
-        rprint(f"[green]Created crawl with {len(url_list)} URLs[/green]", file=sys.stderr)
-
-        # If TTY, show human-readable output
-        if is_tty:
-            rprint(f"  [dim]{crawl.id}[/dim]", file=sys.stderr)
-            for url in url_list[:5]:  # Show first 5 URLs
-                rprint(f"    {url[:70]}", file=sys.stderr)
-            if len(url_list) > 5:
-                rprint(f"    ... and {len(url_list) - 5} more", file=sys.stderr)
-
-        return 0
-
-    except Exception as e:
-        rprint(f"[red]Error creating crawl: {e}[/red]", file=sys.stderr)
-        return 1
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_crawls(
-    status: str | None = None,
-    urls__icontains: str | None = None,
-    max_depth: int | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Crawls as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.crawls.models import Crawl
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Crawl.objects.all().order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "urls__icontains": urls__icontains,
-        "max_depth": max_depth,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for crawl in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(crawl.status, "dim")
-            url_preview = crawl.urls[:50].replace("\n", " ")
-            rprint(f"[{status_color}]{crawl.status:8}[/{status_color}] [dim]{crawl.id}[/dim] {url_preview}...")
-        else:
-            write_record(crawl.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} crawls[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_crawls(
-    status: str | None = None,
-    max_depth: int | None = None,
-) -> int:
-    """
-    Update Crawls from stdin JSONL.
-
-    Reads Crawl records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from django.utils import timezone
-
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.crawls.models import Crawl
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        crawl_id = record.get("id")
-        if not crawl_id:
-            continue
-
-        try:
-            crawl = Crawl.objects.get(id=crawl_id)
-
-            if status:
-                if status not in Crawl.StatusChoices.values:
-                    rprint(f"[red]Invalid crawl status: {status}[/red]", file=sys.stderr)
-                    continue
-                if status == Crawl.StatusChoices.SEALED:
-                    crawl.cancel()
-                elif status == Crawl.StatusChoices.PAUSED:
-                    crawl.pause()
-                elif status == Crawl.StatusChoices.QUEUED:
-                    if crawl.status == Crawl.StatusChoices.PAUSED:
-                        crawl.resume()
-                    else:
-                        crawl.update_and_requeue(status=Crawl.StatusChoices.QUEUED, retry_at=timezone.now())
-                elif status == Crawl.StatusChoices.STARTED:
-                    crawl.update_and_requeue(status=Crawl.StatusChoices.STARTED, retry_at=timezone.now())
-            if max_depth is not None:
-                crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
-                crawl.max_depth = max_depth
-            elif not status:
-                crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
-            updated_count += 1
-
-            if not is_tty:
-                crawl.refresh_from_db()
-                write_record(crawl.to_json())
-
-        except Crawl.DoesNotExist:
-            rprint(f"[yellow]Crawl not found: {crawl_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} crawls[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_crawls(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Crawls from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.crawls.models import Crawl
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    crawl_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not crawl_ids:
-        rprint("[yellow]No valid crawl IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    crawls = Crawl.objects.filter(id__in=crawl_ids)
-    count = crawls.count()
-
-    if count == 0:
-        rprint("[yellow]No matching crawls found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} crawls (dry run)[/yellow]", file=sys.stderr)
-        for crawl in crawls:
-            url_preview = crawl.urls[:50].replace("\n", " ")
-            rprint(f"  [dim]{crawl.id}[/dim] {url_preview}...", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = crawls.delete()
-    rprint(f"[green]Deleted {deleted_count} crawls[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Crawl records."""
-    pass
-
-
-@main.command("create")
-@click.argument("urls", nargs=-1)
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(urls: tuple, depth: int, tag: str, status: str):
-    """Create a Crawl job from URLs or stdin."""
-    sys.exit(create_crawl(urls, depth=depth, tag=tag, status=status))
-
-
-@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--urls__icontains", help="Filter by URLs contains")
-@click.option("--max-depth", type=int, help="Filter by max depth")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(
-    status: str | None,
-    urls__icontains: str | None,
-    max_depth: int | None,
-    limit: int | None,
-):
-    """List Crawls as JSONL."""
-    sys.exit(
-        list_crawls(
-            status=status,
-            urls__icontains=urls__icontains,
-            max_depth=max_depth,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--status", "-s", help="Set status")
-@click.option("--max-depth", type=int, help="Set max depth")
-def update_cmd(status: str | None, max_depth: int | None):
-    """Update Crawls from stdin JSONL."""
-    sys.exit(update_crawls(status=status, max_depth=max_depth))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Crawls from stdin JSONL."""
-    sys.exit(delete_crawls(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_extract.py

@@ -1,203 +0,0 @@
-#!/usr/bin/env python3
-
-"""Run abx-dl snapshot hooks for existing ArchiveBox snapshots."""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox extract"
-
-import sys
-from collections import defaultdict
-from contextlib import redirect_stdout
-
-import rich_click as click
-
-
-def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[str]], set[str]]:
-    """Resolve CLI records to snapshot-level execution requests.
-
-    ArchiveResult input is accepted as a convenient reference to its parent
-    Snapshot and plugin. It is never reset or converted into queued work.
-    """
-    from archivebox.api.v1_core import _uuid_ref_query
-    from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.misc.jsonl import TYPE_ARCHIVERESULT
-
-    explicit_plugins = {name.strip() for name in plugins.split(",") if name.strip()}
-    requested: dict[str, set[str]] = defaultdict(set)
-    missing: set[str] = set()
-
-    for record in records:
-        record_type = record.get("type")
-        record_id = str(record.get("id") or "")
-
-        if record_type == TYPE_ARCHIVERESULT and record.get("snapshot_id"):
-            requested[str(record["snapshot_id"])].update(explicit_plugins or {str(record.get("plugin") or "")})
-            requested[str(record["snapshot_id"])].discard("")
-            continue
-
-        if record_type == TYPE_ARCHIVERESULT:
-            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
-            if result is not None:
-                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
-                continue
-
-        snapshot_id = str(record.get("snapshot_id") or record_id or "")
-        snapshot = Snapshot.objects.filter(id=snapshot_id).only("id").first() if snapshot_id else None
-        if snapshot is None and record.get("url"):
-            snapshot = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").only("id").first()
-        # Bare UUID CLI arguments are parsed as Snapshot records because the
-        # input layer cannot know which model owns them. Preserve the public
-        # convenience of passing an ArchiveResult ID by trying that reference
-        # only after the Snapshot lookup misses.
-        if snapshot is None and record_id and not record.get("url"):
-            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
-            if result is not None:
-                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
-                continue
-        if snapshot is None:
-            missing.add(snapshot_id or str(record.get("url") or ""))
-            continue
-        requested[str(snapshot.id)].update(explicit_plugins)
-
-    return requested, missing
-
-
-def _run_snapshot_requests(requested: dict[str, set[str]], *, wait: bool, show_progress: bool) -> int:
-    from django.utils import timezone
-    from rich import print as rprint
-
-    from archivebox.core.models import Snapshot
-    from archivebox.services.runner import run_crawl
-
-    snapshots = {str(snapshot.id): snapshot for snapshot in Snapshot.objects.filter(id__in=requested).select_related("crawl")}
-    if not snapshots:
-        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
-        return 1
-
-    from archivebox.config.common import get_config
-    from archivebox.plugins.discovery import get_enabled_plugins
-
-    for snapshot_id, plugin_names in requested.items():
-        snapshot = snapshots.get(snapshot_id)
-        if snapshot is not None and not plugin_names:
-            plugin_names.update(get_enabled_plugins(config=get_config(crawl=snapshot.crawl, snapshot=snapshot)))
-
-    if wait and any("search_backend_sonic" in plugin_names for plugin_names in requested.values()):
-        from archivebox.core.takeover_util import ensure_daemon_stack
-
-        with redirect_stdout(sys.stderr):
-            ensure_daemon_stack(reason="Sonic snapshot indexing")
-
-    # Explicit extraction resumes open/paused snapshots at the snapshot level.
-    # Sealed snapshots stay sealed during targeted maintenance backfills.
-    if wait:
-        for snapshot in snapshots.values():
-            if snapshot.status != Snapshot.StatusChoices.SEALED:
-                snapshot.update_and_requeue(
-                    status=Snapshot.StatusChoices.QUEUED,
-                    retry_at=timezone.now(),
-                )
-
-    if not wait:
-        for snapshot_id, plugin_names in requested.items():
-            snapshot = snapshots.get(snapshot_id)
-            if snapshot is None:
-                continue
-            if plugin_names:
-                snapshot.schedule_plugin_run(plugin_names, when=timezone.now())
-            elif snapshot.status == Snapshot.StatusChoices.SEALED:
-                snapshot.update_and_requeue(retry_at=timezone.now())
-            else:
-                snapshot.update_and_requeue(
-                    status=Snapshot.StatusChoices.QUEUED,
-                    retry_at=timezone.now(),
-                )
-        if show_progress:
-            rprint(f"[blue]Queued {len(snapshots)} snapshots for extraction[/blue]", file=sys.stderr)
-        return 0
-
-    grouped: dict[tuple[str, tuple[str, ...]], list[str]] = defaultdict(list)
-    for snapshot_id, plugin_names in requested.items():
-        snapshot = snapshots.get(snapshot_id)
-        if snapshot is not None:
-            grouped[(str(snapshot.crawl_id), tuple(sorted(plugin_names)))].append(snapshot_id)
-
-    for (crawl_id, plugin_names), snapshot_ids in grouped.items():
-        run_crawl(
-            crawl_id,
-            snapshot_ids=sorted(snapshot_ids),
-            selected_plugins=list(plugin_names) or None,
-            show_progress=show_progress,
-        )
-    return 0
-
-
-def run_plugins(
-    args: tuple,
-    records: list[dict] | None = None,
-    plugins: str = "",
-    wait: bool = True,
-    emit_results: bool = True,
-    show_progress: bool = True,
-) -> int:
-    """Execute selected plugins through the snapshot-level runner."""
-    from rich import print as rprint
-
-    from archivebox.core.models import Snapshot
-    from archivebox.misc.jsonl import read_args_or_stdin, write_record
-
-    if records is None:
-        records = list(read_args_or_stdin(args))
-    if not records:
-        rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
-        return 1
-
-    requested, missing = _resolve_requests(records, plugins)
-    for value in sorted(missing):
-        rprint(f"[yellow]Snapshot or ArchiveResult not found: {value}[/yellow]", file=sys.stderr)
-    if not requested:
-        return 1
-
-    exit_code = _run_snapshot_requests(requested, wait=wait, show_progress=show_progress)
-    if exit_code or not emit_results:
-        return exit_code
-
-    is_tty = sys.stdout.isatty()
-    for snapshot in Snapshot.objects.filter(id__in=requested):
-        results = snapshot.archiveresult_set.all()
-        requested_plugins = requested[str(snapshot.id)]
-        if requested_plugins:
-            results = results.filter(plugin__in=requested_plugins)
-        for result in results:
-            if is_tty:
-                color = {"succeeded": "green", "failed": "red", "skipped": "yellow"}.get(result.status, "dim")
-                rprint(f"  [{color}]{result.status}[/{color}] {result.plugin} → {result.output_str or ''}", file=sys.stderr)
-            else:
-                write_record(result.to_json())
-    return 0
-
-
-def process_archiveresult_by_id(archiveresult_id: str) -> int:
-    """Re-run the parent Snapshot plugin referenced by an ArchiveResult."""
-    return run_plugins((), records=[{"id": archiveresult_id}], wait=True)
-
-
-@click.command()
-@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run")
-@click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
-@click.argument("args", nargs=-1)
-def main(plugins: str, wait: bool, args: tuple):
-    """Run plugins on Snapshots; ArchiveResult IDs select their parent plugin."""
-    from archivebox.misc.jsonl import read_args_or_stdin
-
-    records = list(read_args_or_stdin(args))
-    if not records:
-        from rich import print as rprint
-
-        rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
-        sys.exit(1)
-    sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_help.py

@@ -1,158 +1,32 @@
 #!/usr/bin/env python3
-__package__ = "archivebox.cli"
-__command__ = "archivebox help"
 
-import ast
-import importlib.util
-import os
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox help'
+
 import sys
-from datetime import UTC, datetime
-from pathlib import Path
+import argparse
 
-import click
-from rich import print
-from rich.panel import Panel
+from typing import Optional, List, IO
+
+from ..main import help
+from ..util import docstring
+from ..config import OUTPUT_DIR
+from ..logging_util import SmartFormatter, reject_stdin
 
 
-def _command_doc(cmd: str, import_path: str) -> str:
-    def first_doc_line(docstring: str | None) -> str:
-        lines = (docstring or "").splitlines()
-        return lines[0] if lines else ""
-
-    modname, _ = import_path.rsplit(".", 1)
-    spec = importlib.util.find_spec(modname)
-    if spec is None or spec.origin is None:
-        return ""
-
-    try:
-        tree = ast.parse(Path(spec.origin).read_text())
-    except OSError:
-        return ""
-
-    for name in (cmd, "main"):
-        for node in tree.body:
-            if isinstance(node, ast.FunctionDef) and node.name == name:
-                return first_doc_line(ast.get_docstring(node))
-
-    return first_doc_line(ast.get_docstring(tree))
-
-
-def help() -> None:
-    """Print the ArchiveBox help message and usage"""
-
-    from archivebox.cli import ArchiveBoxGroup
-    from archivebox.config import CONSTANTS
-    from archivebox.config.permissions import IN_DOCKER
-
-    version_msg = (
-        f"[dark_magenta]\\[{datetime.now(UTC):%Y-%m-%d %H:%M:%S}][/dark_magenta] "
-        f"[dark_red]ArchiveBox[/dark_red] [dark_goldenrod]v{CONSTANTS.VERSION}[/dark_goldenrod]: "
-        "[green4]archivebox [green3]help"
+@docstring(help.__doc__)
+def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
+    parser = argparse.ArgumentParser(
+        prog=__command__,
+        description=help.__doc__,
+        add_help=True,
+        formatter_class=SmartFormatter,
     )
-    print(Panel(version_msg), file=sys.stderr)
-
-    COMMANDS_HELP_TEXT = (
-        "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.meta_commands.items())
-        + "\n\n    "
-        + "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.setup_commands.items())
-        + "\n\n    "
-        + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.archive_commands.items()
-        )
-    )
-
-    DOCKER_USAGE = (
-        """
-[dodger_blue3]Docker Usage:[/dodger_blue3]
-    [grey53]# using Docker Compose:[/grey53]
-    [blue]docker compose run[/blue] [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-
-    [grey53]# using Docker:[/grey53]
-    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 5797:5797[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-"""
-        if IN_DOCKER
-        else ""
-    )
-    DOCKER_DOCS = (
-        "\n    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker[/link]"
-        if IN_DOCKER
-        else ""
-    )
-    DOCKER_OUTSIDE_HINT = "\n    [grey53]# outside of Docker:[/grey53]" if IN_DOCKER else ""
-    DOCKER_CMD_PREFIX = "[blue]docker ... [/blue]" if IN_DOCKER else ""
-
-    print(f"""{DOCKER_USAGE}
-[deep_sky_blue4]Usage:[/deep_sky_blue4]{DOCKER_OUTSIDE_HINT}
-    [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-
-[deep_sky_blue4]Commands:[/deep_sky_blue4]
-    {COMMANDS_HELP_TEXT}
-
-[deep_sky_blue4]Documentation:[/deep_sky_blue4]
-    [link=https://github.com/ArchiveBox/ArchiveBox/wiki]https://github.com/ArchiveBox/ArchiveBox/wiki[/link]{DOCKER_DOCS}
-    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage[/link]
-    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
-""")
-
-    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
-        pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
-        EXAMPLE_USAGE = f"""
-[light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]
-
-[violet]Hint:[/violet] [i]Common maintenance tasks:[/i]
-    [dark_green]archivebox[/dark_green] [green]init[/green]      [grey53]# make sure database is up-to-date (safe to run multiple times)[/grey53]
-    [dark_green]archivebox[/dark_green] [green]install[/green]   [grey53]# make sure plugins are up-to-date (wget, chrome, singlefile, etc.)[/grey53]
-    [dark_green]archivebox[/dark_green] [green]status[/green]    [grey53]# get a health checkup report on your collection[/grey53]
-    [dark_green]archivebox[/dark_green] [green]update[/green]    [grey53]# retry any previously failed or interrupted archiving tasks[/grey53]
-
-[violet]Hint:[/violet] [i]More example usage:[/i]
-    [dark_green]archivebox[/dark_green] [green]add[/green] --depth=1 "https://example.com/some/page"
-    [dark_green]archivebox[/dark_green] [green]list[/green] --sort=timestamp --csv=timestamp,downloaded_at,url,title
-    [dark_green]archivebox[/dark_green] [green]schedule[/green] --every=day --depth=1 "https://example.com/some/feed.rss"
-    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:5797[/blue]                [grey53]# Start the Web UI / API server[/grey53]
-"""
-        print(
-            Panel(
-                EXAMPLE_USAGE,
-                expand=False,
-                border_style="grey53",
-                title="[green3]:white_check_mark: A collection [light_slate_blue]DATA DIR[/light_slate_blue] is currently active[/green3]",
-                subtitle="Commands run inside this dir will only apply to this collection.",
-            ),
-        )
-    else:
-        DATA_SETUP_HELP = "\n"
-        if IN_DOCKER:
-            DATA_SETUP_HELP += "[violet]Hint:[/violet] When using Docker, you need to mount a volume to use as your data dir:\n"
-            DATA_SETUP_HELP += "    docker run [violet]-v /some/path/data:/data[/violet] archivebox/archivebox ...\n\n"
-        DATA_SETUP_HELP += "To load an [dark_blue]existing[/dark_blue] collection:\n"
-        DATA_SETUP_HELP += "    1. [green]cd[/green] ~/archivebox/data     [grey53]# go into existing [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
-        DATA_SETUP_HELP += f"    2. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# migrate to latest version (safe to run multiple times)[/grey53]\n"
-        DATA_SETUP_HELP += f"    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-update all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
-        DATA_SETUP_HELP += f"    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ...get help with next steps... [/grey53]\n\n"
-        DATA_SETUP_HELP += "To start a [sea_green1]new[/sea_green1] collection:\n"
-        DATA_SETUP_HELP += "    1. [green]mkdir[/green] ~/archivebox/data  [grey53]# create a new, empty [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
-        DATA_SETUP_HELP += "    2. [green]cd[/green] ~/archivebox/data     [grey53]# cd into the new directory[/grey53]\n"
-        DATA_SETUP_HELP += f"    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# initialize ArchiveBox in the new data dir[/grey53]\n"
-        DATA_SETUP_HELP += f"    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-install all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
-        DATA_SETUP_HELP += f"    5. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ... get help with next steps... [/grey53]\n"
-        print(
-            Panel(
-                DATA_SETUP_HELP,
-                expand=False,
-                border_style="grey53",
-                title="[red]:cross_mark: No collection is currently active[/red]",
-                subtitle="All archivebox [green]commands[/green] should be run from inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
-            ),
-        )
+    parser.parse_args(args or ())
+    reject_stdin(__command__, stdin)
+    
+    help(out_dir=pwd or OUTPUT_DIR)
 
 
-@click.command()
-@click.option("--help", "-h", is_flag=True, help="Show help")
-def main(**kwargs):
-    """Print the ArchiveBox help message and usage"""
-    return help()
-
-
-if __name__ == "__main__":
-    main()
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)