Fix comma-separated argument parsing for single Chrome flags with internal commas

2025-12-31 07:36:43 +00:00
1205 changed files with 70473 additions and 195233 deletions
@@ -0,0 +1,31 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(python -m archivebox:*)",
+      "Bash(ls:*)",
+      "Bash(xargs:*)",
+      "Bash(python -c:*)",
+      "Bash(printf:*)",
+      "Bash(pkill:*)",
+      "Bash(python3:*)",
+      "Bash(sqlite3:*)",
+      "WebFetch(domain:github.com)",
+      "Bash(uv add:*)",
+      "Bash(mkdir:*)",
+      "Bash(chmod:*)",
+      "Bash(python -m forum_dl:*)",
+      "Bash(archivebox manage migrate:*)",
+      "Bash(cat:*)",
+      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
+      "Bash(forum-dl:*)",
+      "Bash(pip uninstall:*)",
+      "Bash(python:*)",
+      "Bash(source .venv/bin/activate)",
+      "Bash(mv:*)",
+      "Bash(echo:*)",
+      "Bash(grep:*)",
+      "WebFetch(domain:python-statemachine.readthedocs.io)",
+      "Bash(./bin/run_plugin_tests.sh:*)"
+    ]
+  }
+}
@@ -1,35 +1,17 @@
 .DS_Store
 ._*
-/.heartbeat.json
-/package.json
-/package-lock.json
-__pycache__
-**/__pycache__
 *.pyc
-*.pyo
-*.py[cod]
-*$py.class
+__pycache__/
 .mypy_cache/
 .pytest_cache/
-.ruff_cache/
-.uv-cache/
-.cache/
-.cache*/
 .github/
 .pdm-build/
-.pdm-build/**
 .pdm-python
 .eggs/
-*.egg-info/
 .git/
-!.git/
-.git/*
 .vscode/
 !.git/HEAD
-!.git/packed-refs
-!.git/refs/
-!.git/refs/heads/
-!.git/refs/heads/**
+!.git/refs/heads/*

 venv/
 .venv/
@@ -37,27 +19,9 @@ venv/
 .docker_venv/
 .docker-venv/
 node_modules/
-abx-dl/
-abxpkg/
-abx-plugins/
-abxbus/
 chrome/
 chromeprofile/
 chrome_profile/
-lib/
-out/
-tests/out/
-users/
-archive/
-crawls/
-snapshots/
-logs/
-archivebox-docker-smoke*/
-archivebox-compose-smoke*/
-docker-test/
-docker-test*/
-core
-*.core

 pdm.dev.lock
 pdm.lock
@@ -68,37 +32,14 @@ dist/
 brew_dist/
 deb_dist/
 pip_dist/
-cache/
-caches/
 assets/
 docker/
 website/
 typings/

-# The runtime COPY needs ArchiveBox's package and bin/ entrypoint, not the
-# development UI screenshots or test trees. PNGs are already compressed, so
-# including them creates a large image layer that later cleanup cannot remove.
-publicsite/
-old/
-test/
-tests/
-archivebox/tests/
-
 tmp/
-.tmp/
 data/
 data*/
-
-personas/
-sources/
 output/
 index.sqlite3
 index.sqlite3-wal
-queue.sqlite3
-*.sqlite*
-data.*
-.archivebox_id
-ArchiveBox.conf
-*.stdout
-*.stderr
-*.log
@@ -12,7 +12,7 @@
 - https://github.com/ArchiveBox/ArchiveBox/issues
 - https://github.com/ArchiveBox/ArchiveBox/pulls
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup

 ### Development Setup

@@ -30,8 +30,8 @@ body:
      description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
      render: markdown
      placeholder: |
-        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 5797:5797 archivebox/archivebox` in iTerm2
-        2. Went to the https://127.0.0.1:5797/add/ page in Google Chrome
+        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
+        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
        3. Typed 'https://example.com/xyz' into the 'Add URL' input field
        4. Clicked the 'Add+' button
        5. Got a 500 error and saw the errors below in terminal
@@ -69,7 +69,7 @@ body:
        0.8.6
        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
        
        Binary Dependencies:
@@ -138,7 +138,7 @@ body:
            archivebox:
                image: archivebox/archivebox:latest
                ports:
-                    - 5797:5797
+                    - 8000:8000
                volumes:
                    - ./data:/data
                environment:
@@ -191,6 +191,7 @@ body:
        In the meantime please consider:  
          
          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
          - 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
@@ -57,9 +57,9 @@ body:
  - type: textarea
    id: version
    attributes:
-      label: Share the entire output of the `archivebox version` command for the current version you are using.
+      label: Share the entire output of the `archivebox version` command for the current verison you are using.
      description: |
-        DO NOT JUST ENTER "the latest version" OR YOUR ISSUE WILL BE CLOSED.  
+        DO NOT JUST ENTER "the latest verion" OR YOUR ISSUE WILL BE CLOSED.  
        We need to know what version of ArchiveBox and what feature flags you're currently running with in order to contextualize your feature request.  
        Sometimes we've already fixed the issues in newer BETA versions, sometimes features already exist but may not be available in your specific environment.
          
@@ -71,7 +71,7 @@ body:
        0.8.6
        ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
        IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
        DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
        
        Binary Dependencies:
@@ -109,6 +109,9 @@ body:
          required: false
        - label: "I've had a lot of difficulty getting ArchiveBox set up"
          required: false
+        - label: "I would pay $10/mo for a hosted version of ArchiveBox if it had this feature"
+          required: false
+
  - type: markdown
    attributes:
      value: |
@@ -120,5 +123,6 @@ body:
        In the meantime please consider:  
          
          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 📊 [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
          - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
          - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)
@@ -47,5 +47,6 @@ body:
        In the meantime please consider:  
          
          - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
          - 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
          - ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)
@@ -6,3 +6,6 @@ contact_links:
  - name: 💬 Chat with the dev team & community on Zulip
    url: https://zulip.archivebox.io
    about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
+  - name: 💁‍♂️ Hire us for professional support with fast response times
+    url: https://docs.monadical.com/s/archivebox-consulting-services
+    about: "We provide hosting, develoment, and support, including on-prem/cloud w/ SSO & storage, CAPTCHA-solving, proxies, etc."
@@ -2,15 +2,18 @@

 ---

-## Security Documentation
+## Security Information

-Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
+Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
+
+https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+
+Also see this section of the README about important caveats when running ArchiveBox:
+
+https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
+
+You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:

- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
 - https://github.com/ArchiveBox/ArchiveBox#archive-layout
 - https://github.com/ArchiveBox/ArchiveBox#archivebox-development
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
@@ -20,49 +23,12 @@ Please start by reading these links to understand our existing security model, k

 ## Reporting a Vulnerability

-~We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.~ We no longer offer CVEs or accept private reporting, 95% of the reports we received were low quality AI slop, so now we just ask for normal Github Issues or PRs.
+We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.

-You can contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp) if you have a truly urgent security report, but we do not issue public CVEs.
+1. Go to the Security tab on our Github repo: https://github.com/ArchiveBox/ArchiveBox/security

---
+2. Click the ["Report a Vulnerability"](https://github.com/ArchiveBox/ArchiveBox/security/advisories/new) button

-## CVEs and Report Credits
+3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers

-> [!IMPORTANT]
-> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**  
-> Reporters **will still receive credit in release notes** however.
-
-> [!TIP]
-> ~Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.~
-
-Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
-
---
-
-## Threat Model
-
-### All admin users are assumed to be trusted
-
-ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
-
- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
- non-logged in anonymous guest users
-
-Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
-
-### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
-
-Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
-
-Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
-
-### XSS, CSRF, CORS, CSP
-
-`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
-
-[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
-
-### SSRF / URL Validation
-
-Note archivebox is **designed to be able to archive any URL the server can reach**, including internal intranet URLs, private IPs, and localhost. Users are expected to configure `URL_ALLOWLIST`/`URL_DENYLIST` or proper firewall/ACL-level blocking at the network level if they need to limit what the server should see.
-It does a basic check to try and avoid recursively archiving snapshots hosted by the archivebox server itself, but it's not a strict security check, it's just to prevent wasted CPU/disk from recursively archiving snapshots in a browser that has the ArchiveBox Browser Extension turned on.
+You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).
@@ -1,42 +0,0 @@
-# Stable releases require live dev acceptance
-
-The order is **dev CI → dev image publication → automatic Cabbage deployment →
-real capture and browser acceptance → stable publication**. Never
-substitute a published PyPI wheel, healthy container, or local build for this
-acceptance. The release workflow blocks *before* a stable upload unless Cabbage
-has successful evidence from `staging-acceptance.yml` matching all tracked
-source and dependency pins. Only ArchiveBox's own version fields may change
-between dev and stable.
-
-The staging workflow observes the existing image watcher; it cannot deploy or
-restart the application. On Cabbage, install `bin/staging-acceptance-host.sh`
-as root-owned mode 0755 `/usr/local/sbin/archivebox-staging-acceptance`. Its hash
-is checked against the candidate. A dedicated SSH key must use
-`restrict,command="/usr/local/sbin/archivebox-staging-acceptance"` in authorized
-keys. It accepts only `inspect <source-sha>` and `capture <source-sha>`; it cannot
-run caller-provided commands or archive arbitrary URLs.
-
-The `staging` GitHub environment is restricted to dev/main branches. It contains
-`STAGING_SSH_KEY` and `STAGING_KNOWN_HOSTS`, with independently verified host keys.
-Only trusted release events use the private ugNAS runner route. Never expose
-these credentials to PR jobs or copy an operator's existing root key.
-
-Acceptance creates one tagged background crawl per source through the normal
-add/runner path, enabling every snapshot extractor, including TLSNotary. It
-checks installed pins, completed outputs and their files, unexpected/missing
-plugin results, host OOM kills, and real rendered snapshot pages. For this
-release, the explicitly accepted TLSNotary failures and exact missing-Claude-auth
-diagnostics are retained in the report's `limitations` list; they do not disable
-those plugins or excuse missing/truncated successful outputs. Remove the
-TLSNotary exception when its follow-up fix ships. Logs, result inventories and
-screenshots are retained as Actions artifacts. Other failures remain
-failures; do not publish a success deployment record to bypass them. No code
-mounts are allowed. The Cabbage job must pass before its record authorizes a
-stable upload. Queued capture progress is polled; failed tests are not retried.
-
-Browser acceptance follows the default WACZ viewer into the captured document,
-matching its replay base and original URL and checking that its body is visible
-and contains substantial text. An attached viewer iframe can still be
-`about:blank`, so its presence alone cannot prove replay works. Extracted display
-titles and the captured document's `<title>` may legitimately differ; neither
-title equality nor output thumbnails substitute for checking the actual page.
@@ -1,3 +0,0 @@
-self-hosted-runner:
-  labels:
-    - ugnas-linux
@@ -1,81 +0,0 @@
-{
-  "properties": {
-    "CC_BINARY": {
-      "default": "cc"
-    },
-    "LDAPSEARCH_BINARY": {
-      "default": "ldapsearch"
-    },
-    "INITDB_BINARY": {
-      "default": "initdb"
-    },
-    "PG_CTL_BINARY": {
-      "default": "pg_ctl"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{CC_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "build-essential"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{LDAPSEARCH_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapsearch",
-            "-VV"
-          ]
-        },
-        "apt": {
-          "install_args": [
-            "ldap-utils",
-            "python3-dev",
-            "python3-setuptools",
-            "libssl-dev",
-            "libldap2-dev",
-            "libsasl2-dev",
-            "zlib1g-dev",
-            "libatomic1"
-          ],
-          "version": [
-            "ldapsearch",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ],
-  "postgres_binaries": [
-    {
-      "name": "{INITDB_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "postgresql"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{PG_CTL_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "postgresql"
-          ]
-        }
-      }
-    }
-  ]
-}
@@ -1,13 +0,0 @@
-{
-  "properties": {
-    "BREW_BINARY": {
-      "default": "brew"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{BREW_BINARY}",
-      "binproviders": "env"
-    }
-  ]
-}
@@ -1,44 +0,0 @@
-{
-  "properties": {
-    "CLANG_BINARY": {
-      "default": "clang"
-    },
-    "LDAPVC_BINARY": {
-      "default": "ldapvc"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{CLANG_BINARY}",
-      "binproviders": "env,brew",
-      "overrides": {
-        "brew": {
-          "install_args": [
-            "llvm"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{LDAPVC_BINARY}",
-      "binproviders": "env,brew",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        },
-        "brew": {
-          "install_args": [
-            "openldap"
-          ],
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ]
-}
@@ -1,21 +0,0 @@
-{
-  "properties": {
-    "LDAPVC_BINARY": {
-      "default": "ldapvc"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{LDAPVC_BINARY}",
-      "binproviders": "env",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ]
-}
@@ -1,349 +0,0 @@
-{
-  "properties": {
-    "CURL_BINARY": {
-      "default": "curl"
-    },
-    "BASH_BINARY": {
-      "default": "bash"
-    },
-    "CHMOD_BINARY": {
-      "default": "chmod"
-    },
-    "DOCKER_BINARY": {
-      "default": "docker"
-    },
-    "GIT_BINARY": {
-      "default": "git"
-    },
-    "GREP_BINARY": {
-      "default": "grep"
-    },
-    "GH_BINARY": {
-      "default": "gh"
-    },
-    "EXPORTFS_BINARY": {
-      "default": "exportfs"
-    },
-    "JQ_BINARY": {
-      "default": "jq"
-    },
-    "MKDIR_BINARY": {
-      "default": "mkdir"
-    },
-    "MOUNT_CIFS_BINARY": {
-      "default": "mount.cifs"
-    },
-    "MOUNT_NFS_BINARY": {
-      "default": "mount.nfs"
-    },
-    "NODE_BINARY": {
-      "default": "node"
-    },
-    "PS_BINARY": {
-      "default": "ps"
-    },
-    "PYTHON_BINARY": {
-      "default": "python"
-    },
-    "PV_BINARY": {
-      "default": "pv"
-    },
-    "SSH_BINARY": {
-      "default": "ssh"
-    },
-    "SED_BINARY": {
-      "default": "sed"
-    },
-    "SMBD_BINARY": {
-      "default": "smbd"
-    },
-    "SUDO_BINARY": {
-      "default": "sudo"
-    },
-    "SYSTEMCTL_BINARY": {
-      "default": "systemctl"
-    },
-    "TAR_BINARY": {
-      "default": "tar"
-    },
-    "TREE_BINARY": {
-      "default": "tree"
-    },
-    "TR_BINARY": {
-      "default": "tr"
-    },
-    "RM_BINARY": {
-      "default": "rm"
-    },
-    "UMOUNT_BINARY": {
-      "default": "umount"
-    },
-    "UV_BINARY": {
-      "default": "uv"
-    },
-    "UNAME_BINARY": {
-      "default": "uname"
-    }
-  },
-  "ci_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{JQ_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{UV_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "bsd_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,pyinfra"
-    }
-  ],
-  "freebsd_binaries": [
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env,pyinfra"
-    }
-  ],
-  "docker_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UNAME_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "deploy_binaries": [
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SSH_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "docker_debug_binaries": [
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{PV_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{TAR_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{TREE_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "docker_validation_binaries": [
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GREP_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{MKDIR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{RM_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SED_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SSH_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{TR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UNAME_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "mount_validation_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{CHMOD_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{EXPORTFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "nfs-kernel-server"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{MKDIR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{MOUNT_CIFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "cifs-utils"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{MOUNT_NFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "nfs-common"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{SMBD_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "samba"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SYSTEMCTL_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UMOUNT_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "process_binaries": [
-    {
-      "name": "{PS_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "docs_binaries": [
-    {
-      "name": "{CURL_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "git_binaries": [
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "github_binaries": [
-    {
-      "name": "{GH_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "node_binaries": [
-    {
-      "name": "{NODE_BINARY}",
-      "binproviders": "env,npm",
-      "min_version": "22.23.1",
-      "overrides": {
-        "npm": {
-          "install_args": [
-            "node@22.23.1"
-          ],
-          "postinstall_scripts": true
-        }
-      }
-    }
-  ],
-  "release_binaries": [
-    {
-      "name": "{UV_BINARY}",
-      "binproviders": "env,pip,brew"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{JQ_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{CURL_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ]
-}
@@ -15,7 +15,7 @@ updates:
        patterns:
          - "*"
  - package-ecosystem: "npm"
-    directory: "/etc"
+    directory: "/"
    target-branch: "dev"
    schedule:
      interval: "monthly"
@@ -1,38 +0,0 @@
-# Documentation destinations only: do not crawl archived URLs, app routes, or image assets.
-# Manual run from the repository root with Lychee 0.24.2:
-# git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > /tmp/archivebox-link-inputs.txt
-# lychee --config .github/lychee.toml --root-dir "$PWD" --files-from /tmp/archivebox-link-inputs.txt
-# Explicit tracked inputs include README symlinks and let errors in one file leave others checkable.
-include_fragments = "anchor-only"
-include_verbatim = true
-max_retries = 0
-max_concurrency = 4
-timeout = 20
-no_progress = true
-hidden = true
-extensions = ["md", "html", "py", "rst", "yml", "yaml", "sh", "toml"]
-exclude_path = ['(^|/)(tests|data|apidocs|node_modules|\.git)(/|$)']
-# Includes override this exclude; this is a documentation-link allowlist.
-exclude = ['.*']
-include = [
- '^https://github\.com/ArchiveBox/ArchiveBox/blob/[^/]+/.*\.md(?:#|$)',
- '^https://raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/',
- '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki(?:/|#|$)|/?#)',
- '^https?://docs\.archivebox\.io(?:/|$)',
- '^file:.*\.md(?:#|$)',
- '^file:.*/docs/[^/.#]+(?:#|$)',
-]
-fallback_extensions = ["md"]
-remap = [
- # Lychee's Markdown slugger drops variation selectors. Check these against rendered HTML.
-
- '^file:.*/README\.md(#%EF%B8%8F-cli-usage)$ https://github.com/ArchiveBox/ArchiveBox${1}',
- '^file:.*/docs/Install\.md(#option-a-docker--docker-compose-setup-%EF%B8%8F)$ https://github.com/ArchiveBox/ArchiveBox/wiki/Install${1}',
- # The wiki accepts this case alias, but raw file paths are case-sensitive.
- '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/Setting-up-Storage$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/Setting-Up-Storage.md',
-
- # Bare missing wiki pages silently redirect to Home; raw pages return an actual 404.
- # Keep fragment URLs on the rendered wiki to catch published heading drift.
- '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/([^#/?]+)$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/${1}.md',
-]
-preprocess = { command = "bin/lychee_preprocess.py" }
@@ -1,124 +0,0 @@
-# archivebox.io website
-
-This repository owns its complete website. Build and deploy it without checking
-out another repository's website, fetching shared snippets, or calling a shared
-workflow. Common files are ordinary committed copies; update them deliberately.
-
-## Repository-specific files
-
- Content: `publicsite/index.html`.
- Site/gallery generation: `bin/generate_ui_screenshot_gallery.py`.
- Application capture: `bin/collect_ui_screenshots.sh`.
- Deployment branch: `dev`.
-
-## Common publication contract
-
-1. Build pull requests without deploying them.
-2. Build website pushes immediately, independently of application capture CI.
-3. Restore validated screenshots from trusted default-branch capture jobs.
-   Keep the original capture revision, timestamp, coverage and image checksums.
-4. Retain `site-screenshots` artifacts for 90 days. After expiry, restore the
-   published image set. Never replace a complete gallery with an empty one when
-   restoration fails. Apple device groups are independently validated artifacts;
-   Android's checked-in bootstrap captures retain their local provenance.
-5. Render current content and layout around those captures. Site revision and
-   capture revision are separate facts; reusing captures must not relabel them.
-6. Serialize production restore/build/deploy under one concurrency group and
-   check out the latest default-branch revision when each build starts. Keep the
-   built revision in provenance; later release-bot commits must not block deploys.
-7. Keep platform-specific build, installation, capture, and completeness checks
-   in the application's existing capture workflow. Site-only edits do not need
-   to rerun those jobs.
-
-## Common presentation contract
-
-The base header, Apps menu and icons, footer, brand styles and navigation behavior
-should remain identical copies across the seven product sites. Put this site's
-name, navigation, CTA and extra footer resources outside the common base files.
-Preserve real links, anchor IDs, no-JavaScript navigation, keyboard operation,
-mobile layout, reduced-motion preferences and existing product-specific content.
-Screenshots stay in English. Translation content, when added, lives in separate
-HTML files; no runtime translation service or cross-repository dependency.
-
-## Editing and building
-
-`base/` contains identical independent copies across the seven product sites.
-Common Python scripts share the same behavior; repository formatters may change
-line wrapping without changing their syntax trees. The base owns the shared logo, Apps menu/icons, footer directory, navigation
-behavior and styles. `site.json` and `nav.html` contain this site's name, routes,
-links and CTA. Product content and additional footer resources stay in the native
-HTML/Jinja/Liquid templates listed above. The `ARCHIVEBOX:*` comments are build
-slots; the renderer fills them with static HTML, never browser-side fetches.
-
-The existing builder calls `site.py render` after generating its content. This
-copies `base/chrome.css` and `base/chrome.js` to `site-base/`, assembles the common
-HTML, validates local resources and writes the site revision to root `build.json`.
-Screenshot manifests are left untouched. No build reads another repository.
-
-Common changes are manual copy/paste edits. Compare `base/` and `site.py` with
-another product repository when updating them; do not add a synchronization job,
-package dependency or reusable cross-repository workflow. `nav.html`, `site.json`
-and the marked workflow build section are intentionally repository-specific.
-
-Build: `uv run --no-project python .github/pages/site.py render _site --source publicsite` after restoring screenshots and refreshing the gallery.
-
-Check the built site with real Chromium (also run by Pages CI):
-
-```sh
-export ABXPKG_LIB_DIR=/tmp/archivebox-site-browser
-CHROME_SCHEMA="$(uv run python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
-uv run abxpkg env --install --deps-from="$CHROME_SCHEMA:required_binaries"
-export CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
-uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py SITE_OUTPUT --evidence /tmp/site-evidence
-```
-
-The browser comes from the normal abxpkg provider/cache path; Playwright drives
-that resolved executable instead of installing a second Chromium build.
-Use this workflow's `SITE_OUTPUT` directory. The check covers desktop/mobile
-layout, keyboard dismissal, no-JavaScript links, local resources and visible
-images, and uploads its screenshots as `site-verification` for 14 days.
-
-## Branding and future translations
-
-`base/chrome.css` owns the shared system font stacks, plum accent and CTA shape,
-including the existing `.button` and `.ab-btn` classes. Product styles own their
-layouts, feature sections and platform-specific button states. Shared system
-fonts need no external font downloads. `site.py` emits the same complete SEO/OG/
-Twitter tags everywhere while retaining each native template's title, description,
-canonical URL, locale and social artwork; edit those values with that page's copy.
-
-For translations, keep complete separate HTML files under `es/`, `fr/` and `zh/`,
-and list their output paths in `site.json` alongside the English pages. Share CSS,
-JS, social artwork and the English screenshot gallery. Translate each page's title
-and description, set its own canonical and `lang` (`zh` for Chinese), and add
-reciprocal `hreflang` links only for translations that actually exist. The renderer
-preserves these tags and computes shared asset paths for nested pages. Localized
-header/footer copy and the small language selector can be added with those pages;
-there is no language redirect, placeholder translation or translation runtime now.
-
-## Screenshot strips
-
-`marquee.py` and `base/marquee.css` / `base/marquee.js` are also identical copies.
-Product homepages choose their placement with an `ARCHIVEBOX:MARQUEE` comment.
-The renderer takes real images, captions, dimensions and links from the generated
-`screenshots/index.html`; there is no second screenshot list or browser fetch.
-Desktop/tablet/mobile galleries contribute their desktop views. Apple uses
-`MARQUEE-CLIENT` and `MARQUEE-SERVER` slots to keep its two products separate.
-Catalog sites without a slot do not load the strip assets.
-
-The strip scrolls once, pauses on hover, focus or manual interaction, and has an
-explicit Play/Pause button. Reduced-motion users start paused. Without JavaScript,
-normal image links and horizontal scrolling remain available. Screenshot capture
-manifests and provenance are never rewritten by this presentation step.
-
-Standalone one- or two-image illustrations remain in their authored source.
-Gallery strips with three or more images use the latest restored captures.
-
-## Keeping the common layout in sync
-
-The comments in `base/header.html`, `base/footer-start.html`,
-`base/footer-end.html`, `base/chrome.css`, and `base/chrome.js` list the peer
-repositories. Update those committed copies together when shared branding,
-Apps links, keyboard behavior, or footer resources change. Preserve each site's
-local navigation and content. The former monorepo `site-chrome` generator is
-retired; builds must use only this repository's files.
@@ -1,102 +0,0 @@
-"""Common screenshot artifact transport; coverage checks belong to each app."""
-
-import json
-import subprocess
-from pathlib import PurePosixPath
-from urllib.request import urlopen
-
-
-def api(repo, path):
-    return json.loads(subprocess.check_output(["gh", "api", f"repos/{repo}/{path}"]))
-
-
-def trusted(run, repo, workflow, branch, successful=True):
-    return (
-        (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
-        and run["head_branch"] == branch
-        and run["event"] in ("push", "workflow_dispatch", "schedule")
-        and run["path"] == f".github/workflows/{workflow}"
-        and (not successful or run["conclusion"] == "success")
-    )
-
-
-def runs(repo, workflow, branch, successful=True, artifact_names=("site-screenshots",)):
-    # Ask for capture artifacts directly instead of checking every historical
-    # workflow run (most app runs never produce a complete screenshot set).
-    # Artifact creation time also includes fresh captures from old rerun jobs.
-    candidates = []
-    for name in artifact_names:
-        rows = subprocess.check_output(
-            [
-                "gh",
-                "api",
-                f"repos/{repo}/actions/artifacts?name={name}&per_page=100",
-                "--paginate",
-                "--jq",
-                ".artifacts[] | select(.expired == false) | @json",
-            ],
-            text=True,
-        )
-        candidates.extend(json.loads(row) for row in rows.splitlines())
-    seen = set()
-    for artifact in sorted(candidates, key=lambda item: item["created_at"], reverse=True):
-        source = artifact.get("workflow_run") or {}
-        run_id = source.get("id")
-        if (
-            artifact["name"] not in artifact_names
-            or not run_id
-            or run_id in seen
-            or source.get("head_branch") != branch
-            or source.get("head_repository_id") != source.get("repository_id")
-        ):
-            continue
-        seen.add(run_id)
-        run = api(repo, f"actions/runs/{run_id}")
-        if trusted(run, repo, workflow, branch, successful):
-            yield run
-
-
-def names(repo, run):
-    rows = subprocess.check_output(
-        [
-            "gh",
-            "api",
-            f"repos/{repo}/actions/runs/{run['id']}/artifacts?per_page=100",
-            "--paginate",
-            "--jq",
-            ".artifacts[] | @json",
-        ],
-        text=True,
-    )
-    return {artifact["name"] for row in rows.splitlines() if not (artifact := json.loads(row))["expired"]}
-
-
-def download(repo, run, name, destination):
-    subprocess.run(
-        [
-            "gh",
-            "run",
-            "download",
-            str(run["id"]),
-            "--repo",
-            repo,
-            "--name",
-            name,
-            "--dir",
-            str(destination),
-        ],
-        check=True,
-    )
-
-
-def relative_path(name):
-    path = PurePosixPath(name)
-    if not name or path.is_absolute() or ".." in path.parts or any(c in name for c in ":\\?#%"):
-        raise ValueError(f"Unsafe screenshot path: {name}")
-    return path
-
-
-def fetch(base, name):
-    relative_path(name)
-    with urlopen(base + name, timeout=60) as response:
-        return response.read()
@@ -1,156 +0,0 @@
-/*
-Shared ArchiveBox website navigation styles and behavior.
-Keep this file in sync with .github/pages/base/chrome.css in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
-*/
-/* Common brand tokens. Native templates own product layout and content. */
-:root {
-  --abx-font-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
-  --abx-font-mono: ui-monospace, "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace;
-  --abx-accent: #9b2854;
-  --abx-accent-hover: #782040;
-  --abx-paper: #fcfaf7;
-  --abx-ink: #28252a;
-  --abx-muted: #716c74;
-  --abx-line: #e9e2e3;
-  --font-sans: var(--abx-font-sans);
-  --font-mono: var(--abx-font-mono);
-  --ab-font-sans: var(--abx-font-sans);
-  --ab-font-serif: var(--abx-font-sans);
-  --ab-font-label: var(--abx-font-sans);
-  --ab-font-mono: var(--abx-font-mono);
-  --ab-accent: var(--abx-accent);
-  --ab-accent-hover: var(--abx-accent-hover);
-  --ab-on-accent: #fff;
-}
-body { font-family: var(--abx-font-sans); }
-code, pre, kbd, samp { font-family: var(--abx-font-mono); }
-/* Reuse existing CTA classes; platform selection continues to control colors. */
-.button, .ab-btn {
-  display: inline-flex; align-items: center; justify-content: center; gap: 10px;
-  min-height: 44px; height: auto; max-width: 100%; padding: 10px 20px;
-  border-radius: 24px; font-family: var(--abx-font-sans);
-  font-size: 14px; font-weight: 650; line-height: 1.4;
-  text-align: center; white-space: normal; text-decoration: none;
-}
-.button.primary, .ab-btn--primary { background: var(--abx-accent); border-color: var(--abx-accent); color: #fff; }
-.button.primary:hover, .ab-btn--primary:hover { background: var(--abx-accent-hover); border-color: var(--abx-accent-hover); color: #fff; }
-.button:focus-visible, .ab-btn:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
-.button svg, .button img, .ab-btn svg, .ab-btn img { flex-shrink: 0; }
-@media (prefers-reduced-motion: reduce) { .button, .ab-btn { transition: none; } .button:hover { transform: none; } }
-
-/* Shared ArchiveBox site chrome. Independent copy; edit alongside the other product sites, without a build dependency. */
-/* Native anchor navigation leaves room for the heading and its surrounding content. */
-html { scroll-behavior: smooth; scroll-padding-top: clamp(96px, 25vh, 240px); }
-@media (prefers-reduced-motion: reduce) { html { scroll-behavior: auto; } }
-.abx-header, .abx-footer { box-sizing: border-box; font-family: var(--abx-font-sans); font-size: 14px; line-height: 1.5; letter-spacing: normal; text-align: left; }
-.abx-header *, .abx-footer * { box-sizing: border-box; }
-.abx-header { position: sticky; top: 0; z-index: 100; display: flex; align-items: center; justify-content: space-between; gap: 24px; width: 100%; max-width: none; margin: 0; padding: 15px max(24px, calc((100% - 1280px) / 2)); border: 0; border-bottom: 1px solid #e9e2e3; background: #fcfaf7f5; color: #28252a; backdrop-filter: blur(16px); }
-.abx-header a, .abx-footer a { text-decoration: none; text-underline-offset: .24em; }
-.abx-header .abx-brand, .abx-footer .abx-brand { display: inline-flex; flex: 0 0 auto; align-items: center; gap: 11px; margin: 0; padding: 0; color: inherit; font-size: 19px; font-weight: 750; line-height: 1.2; letter-spacing: -.5px; white-space: nowrap; }
-.abx-brand .abx-logo { display: block; flex: 0 0 auto; width: 32px; height: 32px; margin: 0; }
-.abx-header .abx-subsite { transform: translateY(3px); margin-left: 2px; color: #716c74; font-size: 12px; font-weight: 450; letter-spacing: 0; }
-.abx-header .abx-nav { display: flex; align-items: center; justify-content: flex-end; flex-wrap: wrap; gap: 0; width: auto; max-width: none; margin: 0; padding: 0; font-size: 13px; font-weight: 600; }
-.abx-nav > a, .abx-nav > details { margin: 0 0 0 15px; padding: 0 0 0 15px; border: 0; border-left: 1px solid #ded4d8; border-radius: 0; background: none; }
-.abx-nav > :first-child { margin-left: 0; padding-left: 0; border-left: 0; }
-.abx-header .abx-nav a { display: inline-flex; align-items: center; min-height: 36px; color: #4d424b; white-space: nowrap; }
-.abx-header .abx-nav a:hover { color: #9b2854; }
-.abx-header .abx-nav .abx-cta { margin-left: 18px; padding: 9px 17px; border: 1px solid #9b2854; border-radius: 24px; background: #9b2854; color: #fff; }
-.abx-header .abx-nav .abx-cta:hover { background: #782040; color: #fff; }
-.abx-header .abx-apps { position: relative; }
-.abx-apps > summary { display: flex; align-items: center; gap: 7px; min-height: 36px; list-style: none; cursor: pointer; color: #4d424b; }
-.abx-apps > summary::-webkit-details-marker { display: none; }
-.abx-apps > summary::before { content: '›'; display: inline-block; color: #9b2854; font-size: 22px; line-height: 1; transition: transform .15s; }
-.abx-apps[open] > summary::before { transform: rotate(90deg); }
-.abx-header .abx-app-links { position: absolute; right: 0; top: calc(100% + 12px); width: 440px; max-width: calc(100vw - 24px); max-height: min(70vh, 580px); overflow-y: auto; padding: 8px; border: 1px solid #e9e2e3; border-radius: 16px; background: #fcfaf7; box-shadow: 0 18px 48px #2d152522, 0 3px 8px #2d152508; }
-.abx-header .abx-app-row { display: flex; align-items: center; gap: 8px; min-height: 44px; padding: 4px 6px; border-radius: 10px; transition: background .15s; }
-.abx-header .abx-app-row:hover, .abx-header .abx-app-row:focus-within { background: #f2e8ed; }
-.abx-header .abx-app-links .abx-app-name { flex: 1; min-width: 0; min-height: 34px; padding: 5px 4px; border-radius: 6px; font-size: 13px; line-height: 1.35; white-space: normal; }
-.abx-header .abx-platforms { display: flex; flex: 0 0 auto; align-items: center; gap: 5px; }
-.abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 auto; height: 30px; min-height: 30px; padding: 5px; border: 1px solid #e8dfe3; border-radius: 9px; background: #fffdfb; color: #4d424b; box-shadow: 0 2px 3px #2d152510, inset 0 1px 0 #fff; transition: transform .15s, box-shadow .15s, border-color .15s; }
-.abx-header .abx-app-links .abx-platform { width: 30px; }
-.abx-header .abx-platform svg { display: block; width: 18px; height: 18px; margin: 0; fill: currentColor; }
-.abx-header .abx-app-links .abx-platform-docker { color: #2386ce; }
-.abx-header .abx-app-links .abx-platform-windows, .abx-header .abx-app-links .abx-platform-app-store, .abx-header .abx-app-links .abx-platform-safari { color: #1879c4; }
-.abx-header .abx-app-links .abx-platform-android { color: #29965d; }
-.abx-header .abx-app-links .abx-platform-firefox { color: #e46b24; }
-.abx-header .abx-app-links .abx-platform-brave { color: #e8552b; }
-.abx-header .abx-app-links .abx-platform-edge { color: #168b95; }
-.abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-platform:focus-visible, .abx-header .abx-app-links .abx-tool:hover, .abx-header .abx-app-links .abx-tool:focus-visible { transform: translateY(-2px); border-color: #c99bb0; background: #fff; box-shadow: 0 5px 10px #2d15251c; }
-.abx-header .abx-app-links .abx-tool { padding: 5px 8px; font-size: 11px; white-space: nowrap; }
-.abx-header .abx-tools-row { margin-top: 5px; padding-top: 9px; border-top: 1px solid #e9e2e3; border-radius: 0 0 10px 10px; }
-.abx-header .abx-tools-row .abx-app-name { flex: 0 0 auto; }
-.abx-header .abx-tools-row .abx-platforms { flex: 0 1 auto; flex-wrap: wrap; justify-content: flex-end; margin-left: auto; }
-.abx-header .abx-app-links code { display: inline; vertical-align: baseline; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 1em; font-weight: inherit; line-height: inherit; margin: 0; padding: 0; border: 0; background: transparent; color: inherit; }
-.abx-header .abx-icon-stack { isolation: isolate; gap: 0; padding: 4px 5px; }
-.abx-header .abx-icon-stack .abx-integration { position: relative; z-index: 1; transform: translateY(var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); transition: transform .18s ease, box-shadow .18s ease, border-color .18s ease; }
-.abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -10px; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(2) { --stack-angle: 9deg; --stack-y: -2px; z-index: 2; color: #569a31; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(3) { --stack-angle: -6deg; --stack-y: 2px; z-index: 3; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(4) { --stack-angle: 7deg; --stack-y: -1px; z-index: 4; color: #f00; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(5) { --stack-angle: -10deg; --stack-y: 1px; z-index: 5; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(6) { --stack-angle: 6deg; --stack-y: -2px; z-index: 6; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(7) { --stack-angle: -5deg; z-index: 7; }
-.abx-header .abx-icon-stack .abx-integration:not(:hover, :focus-visible):has(~ .abx-integration:is(:hover, :focus-visible)) { transform: translate(-6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
-.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) ~ .abx-integration:not(:hover, :focus-visible) { transform: translate(6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
-.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { z-index: 20; transform: translateY(-7px) rotate(0deg) scale(1.12); border-color: #c99bb0; box-shadow: 0 8px 14px #2d15252b; }
-@media (hover: none) { .abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -6px; } }
-@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration { transition: none; } }
-.abx-footer { position: relative; z-index: 1; display: block; width: 100%; max-width: none; margin: 72px 0 0; padding: 60px max(24px, calc((100% - 1280px) / 2)) 24px; border: 0; border-top: 3px solid #7e284a; background: radial-gradient(ellipse at 12% 0, #54243b55, transparent 48%), linear-gradient(120deg, #201923, #14151b 75%); color: #ddd2dd; color-scheme: dark; }
-.abx-footer .abx-footer-intro { display: flex; align-items: flex-start; justify-content: space-between; gap: 32px; margin: 0 0 42px; }
-.abx-footer .abx-brand { color: #fff4ed; font-size: 24px; }
-.abx-footer .abx-brand .abx-logo { width: 40px; height: 40px; }
-.abx-footer .abx-tagline { max-width: 430px; margin: 12px 0 0; color: #b8acbb; font-size: 14px; line-height: 1.7; }
-.abx-footer .abx-footer-motto { margin: 2px 0 0; color: #d8b39f; font: italic 25px/1.45 Georgia, serif; text-align: right; }
-.abx-footer .abx-footer-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 42px; margin: 0; }
-.abx-footer .abx-footer-column { margin: 0; padding: 20px 0 0; border: 0; border-top: 1px solid #67505e66; background: none; }
-.abx-footer .abx-footer-column h2 { margin: 0 0 19px; padding: 0; border: 0; color: #e4acbb; font: 650 12px/1.4 var(--abx-font-sans); letter-spacing: .12em; text-transform: uppercase; }
-.abx-footer .abx-footer-column ul { display: block; list-style: none; margin: 0; padding: 0; }
-.abx-footer .abx-footer-column li { display: flex; align-items: center; gap: 10px; margin: 0; padding: 0; min-height: 36px; }
-.abx-footer .abx-footer-column li::before { content: none; }
-.abx-footer .abx-footer-column a { color: #ddd2dd; font-size: 13px; line-height: 1.55; }
-.abx-footer .abx-footer-column a:first-child { padding: 6px 0; }
-.abx-footer a:hover { color: #fff; text-decoration: underline; }
-.abx-footer .abx-source { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 28px; width: 28px; height: 28px; border: 1px solid transparent; border-radius: 50%; color: #a58d9f; }
-.abx-footer .abx-source:hover { color: #f3b5cb; border-color: #80516a; background: #ffffff08; }
-.abx-source svg { width: 15px; height: 15px; fill: currentColor; }
-.abx-footer .abx-footer-bottom { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px 24px; margin-top: 32px; padding-top: 21px; border-top: 1px solid #67505e66; color: #a99bab; font-size: 12px; }
-.abx-footer .abx-footer-bottom p { margin: 0; font-size: inherit; color: inherit; }
-.abx-footer .abx-footer-bottom a { color: #cebbc9; }
-.abx-footer .abx-footer-local { margin: 24px 0 0; padding: 16px 0 0; border: 0; border-top: 1px solid #67505e66; border-radius: 0; background: none; color: #baacbc; font-size: 12px; }
-.abx-footer-local > summary { cursor: pointer; color: #d6bfce; }
-.abx-footer-local > div { padding-top: 18px; }
-.abx-footer-local a { color: #e5aabd; }
-.abx-footer-local .brand, .abx-footer-local .footer-intro, .abx-footer-local .footer-grid, .abx-footer-local .footer-meta { color: inherit; }
-.abx-footer-local .footer-card { background: transparent; border-color: #67505e66; }
-.abx-footer-local h2, .abx-footer-local h3, .abx-footer-local strong { color: #e4d8e3; }
-.abx-footer-local .footer-link-copy span, .abx-footer-local .footer-card-subtitle { color: #baacbc; }
-.abx-header a:focus-visible, .abx-header summary:focus-visible, .abx-footer a:focus-visible, .abx-footer summary:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
-@media (max-width: 1100px) {
-  .abx-header { flex-wrap: wrap; gap: 8px 24px; padding: 12px 20px; }
-  .abx-header .abx-nav { justify-content: flex-start; }
-  .abx-header .abx-app-links { position: fixed; top: auto; left: 20px; right: auto; width: 440px; margin-top: 10px; }
-}
-@media (max-width: 700px) {
-  .abx-header { position: relative; padding: 14px 18px; }
-  .abx-header .abx-brand { flex-wrap: wrap; row-gap: 5px; }
-  .abx-header .abx-nav { width: 100%; gap: 4px 0; font-size: 12px; }
-  .abx-nav > a, .abx-nav > details { margin-left: 10px; padding-left: 10px; }
-  .abx-header .abx-nav .abx-cta { margin-left: 10px; padding: 7px 13px; }
-  .abx-header .abx-app-links { left: 12px; right: 12px; width: auto; max-height: 60vh; }
-  .abx-footer { padding: 40px 22px 22px; margin-top: 48px; }
-  .abx-footer .abx-footer-intro { display: block; margin-bottom: 30px; }
-  .abx-footer .abx-footer-motto { margin-top: 20px; text-align: left; font-size: 22px; }
-  .abx-footer .abx-footer-grid { grid-template-columns: 1fr; gap: 26px; }
-  .abx-footer .abx-footer-column li { min-height: 40px; }
-}
-@media (prefers-reduced-motion: reduce) { .abx-apps > summary::before, .abx-header .abx-app-row, .abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { transition: none; } .abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-tool:hover { transform: none; } }
-
-.abx-footer-local > div > a + a { margin-left: 12px; }
-
-@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { transform: none; } }
@@ -1,28 +0,0 @@
-/*
-Shared ArchiveBox website navigation styles and behavior.
-Keep this file in sync with .github/pages/base/chrome.js in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
-*/
-(() => {
-  const menu = document.querySelector('.abx-apps');
-  document.addEventListener('keydown', event => {
-    if (event.key === 'Escape' && menu?.open) { menu.open = false; menu.querySelector('summary').focus(); }
-  });
-  document.addEventListener('click', event => { if (menu?.open && !menu.contains(event.target)) menu.open = false; });
-  function revealAnchor() {
-    let target;
-    try { target = document.getElementById(decodeURIComponent(window.location.hash.slice(1))); } catch { return; }
-    if (!target) return;
-    let details = target.closest('details');
-    let opened = false;
-    while (details) { if (!details.open) { details.open = true; opened = true; } details = details.parentElement.closest('details'); }
-    if (opened) window.requestAnimationFrame(() => target.scrollIntoView());
-  }
-  window.addEventListener('hashchange', revealAnchor);
-  revealAnchor();
-})();
@@ -1,12 +0,0 @@
-<!--
-Shared ArchiveBox website footer.
-Keep this file in sync with .github/pages/base/footer-end.html in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
-->
-  <div class="abx-footer-bottom"><p>Built in the open. Supported by our community.</p><p><a href="https://github.com/ArchiveBox">GitHub organization</a> · <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Security &amp; privacy</a> · <a href="https://archivebox.io">ArchiveBox.io ↗</a></p></div>
-</footer>
@@ -1,13 +0,0 @@
-<!--
-Shared ArchiveBox website footer.
-Keep this file in sync with .github/pages/base/footer-start.html in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
-->
-<footer class="abx-footer" aria-label="ArchiveBox ecosystem">
-  <div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
-  <div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated
@@ -1,24 +0,0 @@
-<!--
-Shared ArchiveBox website navbar and Apps menu.
-Keep this file in sync with .github/pages/base/header.html in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
-->
-<header class="abx-header">
-  <a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span><span class="abx-subsite">__SITE_NAME__</span></a>
-  <nav class="abx-nav" aria-label="Main navigation">
-    <details class="abx-apps"><summary>Apps</summary><div class="abx-app-links">
-      <div class="abx-app-row"><a class="abx-app-name" href="https://archivebox.io/#install-compose">ArchiveBox Server</a><div class="abx-platforms"><a class="abx-platform abx-platform-docker" href="https://archivebox.io/#install-compose" aria-label="ArchiveBox Server for Docker" title="ArchiveBox Server for Docker"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M349.9 236.3l-66.1 0 0-59.4 66.1 0 0 59.4zm0-204.3l-66.1 0 0 60.7 66.1 0 0-60.7zm78.2 144.8l-66.1 0 0 59.4 66.1 0 0-59.4zM271.8 104.7l-66.1 0 0 60.1 66.1 0 0-60.1zm78.1 0l-66.1 0 0 60.1 66.1 0 0-60.1zm276.8 100c-14.4-9.7-47.6-13.2-73.1-8.4-3.3-24-16.7-44.9-41.1-63.7l-14-9.3-9.3 14c-18.4 27.8-23.4 73.6-3.7 103.8-8.7 4.7-25.8 11.1-48.4 10.7l-434.7 0c-8.7 50.8 5.8 116.8 44 162.1 37.1 43.9 92.7 66.2 165.4 66.2 157.4 0 273.9-72.5 328.4-204.2 21.4 .4 67.6 .1 91.3-45.2 1.5-2.5 6.6-13.2 8.5-17.1l-13.3-8.9zM115.6 176.8l-66 0 0 59.4 66.1 0 0-59.4-.1 0zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm-78.1-72.1l-66.1 0 0 60.1 66.1 0 0-60.1z"/></svg></a><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/#archivebox-serverapp" aria-label="ArchiveBox Server for macOS" title="ArchiveBox Server for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://electron.archivebox.io/" aria-label="ArchiveBox Server for Windows" title="ArchiveBox Server for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://archivebox.io/#install-apt" aria-label="ArchiveBox Server for Linux" title="ArchiveBox Server for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-1Line truncated
-      <div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Desktop</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Desktop for macOS" title="ArchiveBox Desktop for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://windows.archivebox.io/" aria-label="ArchiveBox Desktop for Windows" title="ArchiveBox Desktop for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://linux.archivebox.io/" aria-label="ArchiveBox Desktop for Linux" title="ArchiveBox Desktop for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-10 14.1-10.5 4.7-4.4 13.5-14.2 27.9-14.2 7.1 0 15.6 2.3 25.9 8.9 6.3 4.1 11.3 4.4 22.6 9.3 8.4 3.5 13.7 9.7 10.5 18.2-2.6 7.1-11 14.4-22.7 18.1-11.1 3.6-19.8 16-38.2 14.9-3.9-.2-7-1-9.6-2.1-8-3.5-12.2-10.4-20-15-8.6-4.8-13.2-10.4-14.7-15.3-1.4-4.9 0-9 4.2-12.3zm3.3 334c-2.7 35.1-43.9 34.4-75.3 18-29.9-15.8-68.6-6.5-76.5-21.9-2.4-4.7-2.4-12.7 2.6-26.4l0-.2c2.4-7.6 .6-16-.6-23.9-1.2-7.8-1.8-15 .9-20 3.5-6.7 8.5-9.1 14.8-11.3 10.3-3.7 11.8-3.4 19.6-9.9 5.5-5.7 9.5-12.9 14.3-18 5.1-5.5 10-8.1 17.7-6.9 8.1 1.2 15.1 6.8 21.9 16l19.6 35.6c9.5 19.9 43.1 48.4 41 68.9zm-1.4-25.9c-4.1-6.6-9.6-13.6-14.4-19.6 7.1 0 14.2-2.2 16.7-8.9 2.3-6.2 0-14.9-7.4-24.9-13.5-18.2-38.3-32.5-38.3-32.5-13.5-8.4-21.1-18.7-24.6-29.9s-3-23.3-.3-35.2c5.2-22.9 18.6-45.2 27.2-59.2 2.3-1.7 .8 3.2-8.7 20.8-8.5 16.1-24.4 53.3-2.6 82.4 .6-20.7 5.5-41.8 13.8-61.5 12-27.4 37.3-74.9 39.3-112.7 1.1 .8 4.6 3.2 6.2 4.1 4.6 2.7 8.1 6.7 12.6 10.3 12.4 10 28.5 9.2 42.4 1.2 6.2-3.5 11.2-7.5 15.9-9 9.9-3.1 17.8-8.6 22.3-15 7.7 30.4 25.7 74.3 37.2 95.7 6.1 11.4 18.3 35.5 23.6 64.6 3.3-.1 7 .4 10.9 1.4 13.8-35.7-11.7-74.2-23.3-84.9-4.7-4.6-4.9-6.6-2.6-6.5 12.6 11.2 29.2 33.7 35.2 59 2.8 11.6 3.3 23.7 .4 35.7 16.4 6.8 35.9 17.9 30.7 34.8-2.2-.1-3.2 0-4Line truncated
-      <div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Mobile</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Mobile for iOS" title="ArchiveBox Mobile for iOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-android" href="https://android.archivebox.io/" aria-label="ArchiveBox Mobile for Android" title="ArchiveBox Mobile for Android"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!-- Font Awesome Free 5.0.13 by Fonticons, Inc. https://fontawesome.com License: https://creativecommons.org/licenses/by/4.0/ --><path d="M89.6 204.5v115.8c0 15.4-12.1 27.7-27.5 27.7-15.3 0-30.1-12.4-30.1-27.7V204.5c0-15.1 14.8-27.5 30.1-27.5 15.1 0 27.5 12.4 27.5 27.5zm10.8 157c0 16.4 13.2 29.6 29.6 29.6h19.9l.3 61.1c0 36.9 55.2 36.6 55.2 0v-61.1h37.2v61.1c0 36.7 55.5 36.8 55.5 0v-61.1h20.2c16.2 0 29.4-13.2 29.4-29.6V182.1H100.4v179.4zm248-189.1H99.3c0-42.8 25.6-80 63.6-99.4l-19.1-35.3c-2.8-4.9 4.3-8 6.7-3.8l19.4 35.6c34.9-15.5 75-14.7 108.3 0L297.5 34c2.5-4.3 9.5-1.1 6.7 3.8L285.1 73c37.7 19.4 63.3 56.6 63.3 99.4zm-170.7-55.5c0-5.7-4.6-10.5-10.5-10.5-5.7 0-10.2 4.8-10.2 10.5s4.6 10.5 10.2 10.5c5.9 0 10.5-4.8 10.5-10.5zm113.4 0c0-5.7-4.6-10.5-10.2-10.5-5.9 0-10.5 4.8-10.5 10.5s4.6 10.5 10.5 10.5c5.6 0 10.2-4.8 10.2-10.5zm94.8 60.1c-15.1 0-27.5 12.1-27.5 27.5v115.8c0 15.4 12.4 27.7 27.5 27.7 15.4 0 30.1-12.4 30.1-27.7V204.5c0-15.4-14.8-27.5-30.1-27.5z"/></svg></a></div></div>
-      <div class="abx-app-row"><a class="abx-app-name" href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><div class="abx-platforms"><a class="abx-platform abx-platform-chrome" href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj" aria-label="ArchiveBox Browser Extension for Chrome" title="ArchiveBox Browser Extension for Chrome"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc. --><path d="M0 256C0 209.4 12.47 165.6 34.27 127.1L144.1 318.3C166 357.5 207.9 384 256 384C270.3 384 283.1 381.7 296.8 377.4L220.5 509.6C95.9 492.3 0 385.3 0 256zM365.1 321.6C377.4 302.4 384 279.1 384 256C384 217.8 367.2 183.5 340.7 160H493.4C505.4 189.6 512 222.1 512 256C512 397.4 397.4 511.1 256 512L365.1 321.6zM477.8 128H256C193.1 128 142.3 172.1 130.5 230.7L54.19 98.47C101 38.53 174 0 256 0C350.8 0 433.5 51.48 477.8 128V128zM168 256C168 207.4 207.4 168 256 168C304.6 168 344 207.4 344 256C344 304.6 304.6 344 256 344C207.4 344 168 304.6 168 256z"/></svg></a><a class="abx-platform abx-platform-safari" href="https://extension.archivebox.io/" aria-label="ArchiveBox Browser Extension for Safari" title="ArchiveBox Browser Extension for Safari"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M274.7 274.7L237.3 237.3 166 346 274.7 274.7zM256 8a248 248 0 1 0 0 496 248 248 0 1 0 0-496zM411.8 182.8l14.8-6.1c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.8 1 2.6 1.7 1.3 1.6 1.7 2.6 .6 2 .6 3.1-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.3-2.6 1.7L418 197.6c-2 .8-4.2 .8-6.1 0s-3.5-2.4-4.3-4.3-.8-4.2 0-6.1 2.4-3.5 4.3-4.3zM314.4 94l6.1-14.8c.8-2 2.4-3.5 4.3-4.3s4.2-.8 6.1 0 3.5 2.4 4.3 4.3 .8 4.2 0 6.1l-6.1 14.8c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0c-1-.4-1.9-1-2.6-1.7s-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1zM256 60c2.1 0 4.2 .8 5.7 2.3S264 65.9 264 68l0 16c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7S253.9 60 256 60zM181 74.9c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.9 1 2.6 1.7 1.3 1.6 1.7 2.6L197.6 94c.4 1 .7 2 .7 3.1s-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.4-2.6 1.8-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.8-1.3-1.7-1.7-2.7l-6.1-14.8c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3zm-63.6 42.5c1.5-1.5 3.5-2.3 5.7-2.3s4.2 .8 5.7 2.3L140 128.7c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3l-11.3-11.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6l0 0zM60 256c0-2.1 .8-4.2 2.3-5.7S65.9 248 68 248l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3l-16 0c-2.1 0-4.2-.8-5.7-2.3S60 258.1 60 256zm40.2 73.2l-14.8 6.1c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1 1-1.9 1.7-2.6 1.6-1.3 2.6-1.7L94 314.4c2-.8 4.2-.8 6.1 0s3.5 2.4 4.3 4.3 .8 4.2 0 6.1-2.4 3.5-4.3 4.3zm4.3-136c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1 2.4-3.5 4.3-4.3 4.2-.8 6.1 0l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1l0 0zM197.6 418l-6.1 14.8c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1l6.1-14.8c.4-1 1-1.9 1.7-2.7s1.6-1.4 2.6-1.8 2-.6 3.1-.6 2.1 .2 3.1 .6 1.9 1 2.6 1.8 1.3 1.7 1.7 2.6 .6 2 .6 3.1-.2 2.1-.7 3.1zM264 444c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7s3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3 2.3 3.5 2.3 5.7l0 16zm67-6.9c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6L314.4 418c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3 4.2-.8 6.1 0 3.5 2.4 4.3 4.3l6.1 14.8c.8 2 .8 4.2 0 6.1s-2.4 3.5-4.3 4.3zm63.6-42.5c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3L372 383.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6 3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3l11.3 11.3c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6l0 0zM286.2 286.2L110.3 401.7 225.8 225.8 401.7 110.3 286.2 286.2zM437.1 331c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1c.4-1 1-1.9 1.7-2.6s1.6-1.3 2.6-1.7 2-.6 3.1-.6 2.1 .2 3.1 .6l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1zm6.9-67l-16 0c-2.1 0-4.2-.8-5.7-2.3s-2.3-3.5-2.3-5.7 .8-4.2 2.3-5.7 3.5-2.3 5.7-2.3l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3z"/></svg></a><a class="abx-platform abx-platform-firefox" href="https://addons.mozilla.org/firefox/addon/archivebox-exporter/" aria-label="ArchiveBox Browser Extension for Firefox" title="ArchiveBox Browser Extension for Firefox"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512Line truncated
-      <div class="abx-app-row abx-plugin-row"><a class="abx-app-name" href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><div class="abx-platforms abx-icon-stack"><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Google Drive storage" title="Google Drive storage"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="m38.7 419.3 22.6 39c4.7 8.2 11.4 14.7 19.4 19.4l80.6-139.6H0c0 9.1 2.3 18.2 7 26.4z" style="fill:#0066da"/><path d="M256 173.9 175.4 34.3c-7.9 4.7-14.7 11.1-19.4 19.4L7 311.7c-4.6 8-7 17.1-7 26.4h161.3z" style="fill:#00ac47"/><path d="M431.4 477.7c7.9-4.7 14.7-11.1 19.4-19.4l9.4-16.1 44.9-77.7c4.7-8.2 7-17.3 7-26.4H350.7l34.3 67.4z" style="fill:#ea4335"/><path d="m256 173.9 80.6-139.6c-7.9-4.7-17-7-26.4-7H201.8c-9.4 0-18.5 2.6-26.4 7z" style="fill:#00832d"/><path d="M350.7 338.1H161.3L80.6 477.7c7.9 4.7 17 7 26.4 7h298c9.4 0 18.5-2.6 26.4-7z" style="fill:#2684fc"/><path d="M430.5 182.7 356 53.7c-4.7-8.2-11.4-14.7-19.4-19.4L256 173.9l94.7 164.2h161c0-9.1-2.3-18.2-7-26.4z" style="fill:#ffba00"/></svg></a><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Amazon S3 storage" title="Amazon S3 storage"><svg aria-hidden="true" focusable="false" fill="#569A31" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M20.913 13.147l.12-.895c.947.576 1.258.922 1.354 1.071-.16.031-.562.046-1.474-.176zm-2.174 7.988a.547.547 0 0 0-.005.073c0 .084-.207.405-1.124.768a10.28 10.28 0 0 1-1.438.432c-1.405.325-3.128.504-4.853.504-4.612 0-7.412-1.184-7.412-1.704a.547.547 0 0 0-.005-.073L1.81 5.602c.135.078.28.154.432.227.042.02.086.038.128.057.134.062.272.122.417.18l.179.069c.154.058.314.114.478.168.043.013.084.029.13.043.207.065.423.127.646.187l.176.044c.175.044.353.087.534.127a23.414 23.414 0 0 0 .843.17l.121.023c.252.045.508.085.768.122.071.011.144.02.216.03.2.027.4.053.604.077l.24.027c.245.026.49.05.74.07l.081.009c.275.022.552.04.83.056l.233.012c.21.01.422.018.633.025a33.088 33.088 0 0 0 2.795-.026l.232-.011c.278-.016.555-.034.83-.056l.08-.008c.25-.02.497-.045.742-.072l.238-.026c.205-.024.408-.05.609-.077.07-.01.141-.019.211-.03.261-.037.519-.078.772-.122l.111-.02c.215-.04.427-.082.634-.125l.212-.047c.186-.041.368-.085.546-.13l.166-.042c.225-.06.444-.122.654-.189.04-.012.077-.026.115-.038a10.6 10.6 0 0 0 .493-.173c.058-.021.114-.044.17-.066.15-.06.293-.12.43-.185.038-.017.079-.034.116-.052.153-.073.3-.15.436-.228l-.976 7.245c-2.488-.78-5.805-2.292-7.311-3a1.09 1.09 0 0 0-1.088-1.085c-.6 0-1.088.489-1.088 1.088 0 .6.488 1.089 1.088 1.089.196 0 .378-.056.537-.148 1.72.812 5.144 2.367 7.715 3.15zm-7.42-20.047c5.677 0 9.676 1.759 9.75 2.736l-.014.113c-.01.033-.031.067-.048.101-.015.028-.026.057-.047.087-.024.033-.058.068-.09.102-.028.03-.051.06-.084.09-.038.035-.087.07-.133.105-.04.03-.074.06-.119.091-.053.036-.116.071-.177.107-.05.03-.095.06-.15.09-.068.036-.147.073-.222.11-.059.028-.114.057-.177.085-.084.038-.177.074-.268.111-.068.027-.13.054-.203.082-.097.036-.205.072-.31.107-.075.026-.148.053-.228.079-.111.035-.233.069-.35.103-.085.024-.165.05-.253.073-.124.034-.258.065-.389.098-.093.022-.181.046-.278.068-.139.032-.287.061-.433.091-.098.02-.191.041-.293.06-.155.03-.32.057-.482.084-.1.018-.198.036-.302.052-.166.026-.342.048-.515.072-.11.014-.213.03-.325.044-.181.023-.372.041-.56.06-.11.012-.218.025-.332.036-.188.016-.386.029-.58.043-.122.009-.24.02-.364.028-.207.012-.422.02-.635.028-.12.005-.234.012-.354.016a35.605 35.605 0 0 1-2.069 0c-.12-.004-.234-.011-.352-.016-.214-.008-.43-.016-.637-.028-.122-.008-.238-.02-.36-.027-.195-.015-.394-.028-.584-.044-.11-.01-.215-.024-.324-.035-.19-.02-.384-.038-.568-.06l-.315-.044c-.176-.024-.355-.046-.525-.073-.1-.015-.192-.033-.29-.05-.167-.028-.335-.055-.494-.086-.096-.018-.183-.038-.276-.056-.151-.032-.305-.062-.45-.095-.09-.02-.173-.043-.26-.064-.138-.034-.277-.067-.407-.102-.082-.022-.157-.046-.235-.069a11.75 11.75 0 0 1-.368-.108c-.075-.024-.141-.049-.213-.073-.11-.037-.223-.075-.325-.113-.067-.025-.125-.051-.188-.077-.096-.038-.195-.076-.282-.115-.06-.027-.11-.054-.166-.08-.08-.039-.162-.077-.233-.116-.052-.028-.094-.055-.142-.084-.063-.038-.13-.075-.185-.113-.043-.029-.075-.058-.113-.086-.048-.037-.098-.073-.139-.11-.032-.029-.054-.057-.08-.087-.033-.035-.069-.07-.093-.104-.02-.03-.031-.058-.046-.086-.018-.035-.039-.068-.049-.102l-.015-.113c.076-.977 4.074-2.736 9.748-2.736zm12.182 12.124c-.118-.628-.84-1.291-2.31-2.128l.963-7.16a.531.531 0 0 0 .005-.073C22.16 1.581 16.447 0 11.32 0 6.194 0 .482 1.581.482 3.851a.58.58 0 0 0 .005.072L2.819 21.25c.071 2.002 5.236 2.75 8.5 2.75 1.805 0 3.615-.188 5.098-.531.598-.138 1.133-.3 1.592-.48 1.18-.467 1.789-1.053 1.813-1.739l.945-7.018c.557.131 1.016.197 1.389.197.54 0 .902-.137 1.134-.413aLine truncated
-      <div class="abx-app-row abx-tools-row"><a class="abx-app-name" href="https://abx-dl.archivebox.io/">Tools</a><div class="abx-platforms"><a class="abx-tool" href="https://abx-dl.archivebox.io/"><span>One-shot <code>abx-dl</code> CLI</span></a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/tree/dev/archivebox/mcp">MCP</a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-api-usage">API</a><a class="abx-tool" href="https://web.archivebox.io">Demo</a></div></div>
-    </div></details>
-    __SITE_NAV__
-  </nav>
-</header>
@@ -1,11 +0,0 @@
-/* Identical standalone gallery strip across ArchiveBox product sites. */
-/* The root query container measures the usable width, excluding scrollbars. */
-html { container-type: inline-size; }
-.abx-marquee { min-width: 0; width: 100cqw; max-width: none; margin: 24px calc(50% - 50cqw) 40px; padding: 0; overflow: hidden; }
-.abx-marquee .abx-marquee-viewport { width: 100%; overflow-x: auto; scroll-behavior: auto; overscroll-behavior-x: contain; scrollbar-width: thin; padding: 16px 24px 28px; box-sizing: border-box; }
-.abx-marquee .abx-marquee-track { display: flex; align-items: flex-start; gap: 24px; width: max-content; margin-inline: auto; }
-.abx-marquee .abx-marquee-card { display: block; flex: none; width: min-content; overflow: hidden; border-radius: 10px; background: #fff; box-shadow: 0 0 0 1px rgb(0 0 0 / 12%), 0 2px 4px rgb(0 0 0 / 10%), 0 8px 16px rgb(0 0 0 / 16%); text-decoration: none; color: #533344; }
-.abx-marquee .abx-marquee-card img { display: block; height: clamp(240px, 30vw, 400px); width: auto; max-width: none; max-height: none; object-fit: contain; box-shadow: none; border: 0; border-radius: 0; background: transparent; padding: 0; margin: 0; }
-.abx-marquee :focus-visible { outline: 2px solid #9b2854; outline-offset: 3px; }
-.abx-marquee .abx-marquee-viewport:focus-visible { outline-offset: -3px; }
-@media (max-width: 600px) { .abx-marquee .abx-marquee-track { gap: 16px; } .abx-marquee .abx-marquee-card img { height: 240px; } }
@@ -1,44 +0,0 @@
-/* Scroll real gallery links once; keep them usable without JavaScript. */
-(() => {
-  document.querySelectorAll('.abx-marquee').forEach(gallery => {
-    const viewport = gallery.querySelector('.abx-marquee-viewport');
-    const reducedMotion = window.matchMedia('(prefers-reduced-motion: reduce)');
-    let paused = reducedMotion.matches;
-    let hovering = false;
-    let focused = false;
-    let visible = false;
-    let frame;
-    let previous;
-    let offset = viewport.scrollLeft;
-    function tick(time) {
-      const distance = viewport.scrollWidth - viewport.clientWidth;
-      if (distance <= 0) return;
-      if (previous !== undefined) {
-        offset = Math.min(offset + Math.min(time - previous, 50) * 0.035, distance);
-        viewport.scrollTo({ left: offset, behavior: 'instant' });
-      }
-      if (offset >= distance - 1) { paused = true; return; }
-      previous = time;
-      frame = window.requestAnimationFrame(tick);
-    }
-    function sync() {
-      window.cancelAnimationFrame(frame);
-      previous = undefined;
-      offset = viewport.scrollLeft;
-      if (!paused && !hovering && !focused && visible && !document.hidden) frame = window.requestAnimationFrame(tick);
-    }
-    viewport.addEventListener('pointerenter', event => { if (event.pointerType === 'mouse') { hovering = true; sync(); } });
-    viewport.addEventListener('pointerleave', () => { hovering = false; sync(); });
-    viewport.addEventListener('focusin', () => { focused = true; sync(); });
-    viewport.addEventListener('focusout', event => { focused = viewport.contains(event.relatedTarget); sync(); });
-    function pauseForInteraction() { paused = true; sync(); }
-    viewport.addEventListener('pointerdown', pauseForInteraction);
-    viewport.addEventListener('wheel', pauseForInteraction, { passive: true });
-    viewport.addEventListener('keydown', pauseForInteraction);
-    reducedMotion.addEventListener('change', () => { paused = reducedMotion.matches; sync(); });
-    document.addEventListener('visibilitychange', sync);
-    new window.IntersectionObserver(([entry]) => { visible = entry.isIntersecting; sync(); }).observe(gallery);
-    new window.ResizeObserver(sync).observe(viewport);
-    viewport.querySelectorAll('img').forEach(image => image.addEventListener('load', sync));
-  });
-})();
@@ -1,96 +0,0 @@
-"""Render homepage screenshot strips from the site's existing gallery HTML."""
-
-import re
-from html import escape
-from html.parser import HTMLParser
-from urllib.parse import urljoin
-
-
-class Gallery(HTMLParser):
-    """Only gallery figures are captures; navigation and decorative images are not."""
-
-    def __init__(self, base):
-        super().__init__()
-        self.base = base
-        self.stack = []
-        self.captures = []
-
-    def handle_starttag(self, tag, attrs):
-        attrs = dict(attrs)
-        if tag == "img" and any(name == "figure" for name, _ in self.stack):
-            context = [item for _, item in self.stack]
-            profile = next(
-                (
-                    item.get("data-profile") or item.get("data-viewport")
-                    for item in reversed(context)
-                    if item.get("data-profile") or item.get("data-viewport")
-                ),
-                None,
-            )
-            if profile not in (None, "desktop"):
-                return
-            source = attrs.get("src", "")
-            if not source or not attrs.get("alt"):
-                return
-            anchor = next((item["id"] for name, item in reversed(self.stack) if name in ("article", "figure") and item.get("id")), "")
-            server = any(item.get("data-platform") == "server" for item in context)
-            self.captures.append(
-                {
-                    "src": urljoin(self.base, source),
-                    "href": self.base + "#" + anchor if anchor else urljoin(self.base, source),
-                    "title": attrs["alt"],
-                    "width": attrs.get("width", ""),
-                    "height": attrs.get("height", ""),
-                    "product": "server" if server else "client",
-                },
-            )
-        if tag not in {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}:
-            self.stack.append((tag, attrs))
-
-    def handle_endtag(self, tag):
-        for index in range(len(self.stack) - 1, -1, -1):
-            if self.stack[index][0] == tag:
-                del self.stack[index:]
-                break
-
-
-def render(output, baseurl="", featured=()):
-    homepage = output / "index.html"
-    text = homepage.read_text()
-    slots = re.findall(r"<!-- ARCHIVEBOX:(MARQUEE(?:-CLIENT|-SERVER)?) -->", text)
-    if not slots:
-        return False
-    base = "/" + baseurl.strip("/") + "/" if baseurl.strip("/") else "/"
-    gallery = Gallery(base + "screenshots/")
-    gallery.feed((output / "screenshots/index.html").read_text())
-    for slot in slots:
-        product = slot.removeprefix("MARQUEE-").lower() if "-" in slot else None
-        captures = [capture for capture in gallery.captures if product is None or capture["product"] == product]
-        priority = {title: index for index, title in enumerate(featured)}
-        captures.sort(key=lambda capture: priority.get(capture["title"], len(priority)))
-        cards = []
-        for capture in captures:
-            dimensions = "".join(f' {key}="{escape(capture[key])}"' for key in ("width", "height") if capture[key])
-            cards.append(
-                f'<a class="abx-marquee-card" href="{escape(capture["href"])}"><img src="{escape(capture["src"])}" alt="{escape(capture["title"])}"{dimensions} loading="lazy" decoding="async"></a>',
-            )
-        viewport_id = slot.lower()
-        content = (
-            f'<section class="abx-marquee" aria-label="App screenshots"><div class="abx-marquee-viewport" id="{viewport_id}" tabindex="0" aria-label="Screenshots. Scroll sideways to explore."><div class="abx-marquee-track">{"".join(cards)}</div></div></section>'
-            if cards
-            else ""
-        )
-        start, end = f"<!-- ARCHIVEBOX:{slot} -->", f"<!-- /ARCHIVEBOX:{slot} -->"
-        replacement = start + "\n" + content + "\n" + end
-        if end in text:
-            text = re.sub(
-                re.escape(start) + r".*?" + re.escape(end),
-                lambda _, replacement=replacement: replacement,
-                text,
-                count=1,
-                flags=re.DOTALL,
-            )
-        else:
-            text = text.replace(start, replacement, 1)
-    homepage.write_text(text)
-    return True
@@ -1,4 +0,0 @@
-<!-- Repository-specific navigation and primary CTA. -->
-<a href="__SITE_BASE__#use-cases">Use cases</a><a href="__SITE_BASE__#configuration">Configuration</a><a href="__SITE_BASE__#layout-security">Layout &amp; security</a><a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
-    <a href="https://github.com/ArchiveBox/ArchiveBox">GitHub ↗</a>
-    <a class="abx-cta" href="__SITE_BASE__#install">Get started</a>
@@ -1,16 +0,0 @@
-{
-  "name": "Web Archive",
-  "url": "https://archivebox.io/",
-  "pages": [
-    "index.html",
-    "screenshots/index.html"
-  ],
-  "marquee_featured": [
-    "Snapshots grid — desktop",
-    "Snapshot View (header collapsed) — desktop",
-    "AI agent — desktop",
-    "Archive results — desktop",
-    "Add URLs — desktop",
-    "Snapshots table — desktop"
-  ]
-}
@@ -1,215 +0,0 @@
-"""Assemble and verify this repository's static Pages site (standard library only)."""
-
-import argparse
-import json
-import re
-import shutil
-import subprocess
-from datetime import UTC, datetime
-from html import escape, unescape
-from html.parser import HTMLParser
-from pathlib import Path
-from urllib.parse import unquote, urlsplit
-
-from marquee import render as render_marquee
-
-HERE = Path(__file__).resolve().parent
-ROOT = HERE.parents[1]
-
-
-def replace_block(text, name, content):
-    start, end = f"<!-- ARCHIVEBOX:{name} -->", f"<!-- /ARCHIVEBOX:{name} -->"
-    pattern = re.escape(start) + r".*?" + re.escape(end)
-    if re.search(pattern, text, re.DOTALL):
-        return re.sub(
-            pattern,
-            lambda _: f"{start}\n{content}\n{end}",
-            text,
-            count=1,
-            flags=re.DOTALL,
-        )
-    if text.count(start) != 1:
-        raise ValueError(f"Expected one {name} placeholder")
-    return text.replace(start, f"{start}\n{content}\n{end}", 1)
-
-
-def render_metadata(text):
-    """Keep product copy in native templates; render one consistent SEO block."""
-    page = Page()
-    page.feed(text)
-    title = unescape(re.search(r"<title>(.*?)</title>", text, re.DOTALL)[1])
-    meta = page.metadata
-    description = meta.get("description", meta.get("og:description", ""))
-    image = meta.get("og:image", meta.get("twitter:image", ""))
-    if not description or not image or not page.canonical:
-        raise ValueError("Each page needs a description, social image and canonical")
-    values = {
-        "description": description,
-        "robots": "index,follow,max-image-preview:large",
-        "theme-color": "#9b2854",
-        "og:type": "website",
-        "og:site_name": meta.get("og:site_name", "ArchiveBox"),
-        "og:locale": meta.get("og:locale", "en_US"),
-        "og:url": page.canonical,
-        "og:title": title,
-        "og:description": description,
-        "og:image": image,
-        "og:image:type": meta.get("og:image:type", "image/png"),
-        "og:image:width": meta.get("og:image:width", "1200"),
-        "og:image:height": meta.get("og:image:height", "630"),
-        "og:image:alt": meta.get("og:image:alt", title),
-        "twitter:card": "summary_large_image",
-        "twitter:site": "@ArchiveBoxApp",
-        "twitter:title": title,
-        "twitter:description": description,
-        "twitter:image": image,
-        "twitter:image:alt": meta.get("og:image:alt", title),
-    }
-
-    def strip_managed_tag(match):
-        tag = Page()
-        tag.feed(match[0])
-        if tag.canonicals or any(key in values for key in tag.metadata):
-            return ""
-        # System fonts are shared across sites, without a network font dependency.
-        if "fonts.googleapis.com" in match[0] or "fonts.gstatic.com" in match[0]:
-            return ""
-        return match[0]
-
-    text = re.sub(r"<(?:meta|link)\b[^>]*>", strip_managed_tag, text)
-    text = re.sub(r"<title>.*?</title>", "", text, flags=re.DOTALL)
-    tags = [
-        f"<title>{escape(title)}</title>",
-        f'<link rel="canonical" href="{escape(page.canonical, quote=True)}">',
-    ]
-    for key, value in values.items():
-        attr = "property" if key.startswith("og:") else "name"
-        tags.append(f'<meta {attr}="{key}" content="{escape(value, quote=True)}">')
-    if "<!-- ARCHIVEBOX:METADATA -->" not in text:
-        text = text.replace("</head>", "<!-- ARCHIVEBOX:METADATA -->\n</head>", 1)
-    return replace_block(text, "METADATA", "\n".join(tags))
-
-
-def render(output, source=None, baseurl=""):
-    config = json.loads((HERE / "site.json").read_text())
-    output = output.resolve()
-    if source:
-        source = source.resolve()
-        if output == source or output in source.parents or source in output.parents:
-            raise ValueError("Build output must be separate from the source directory")
-        if output == ROOT or output in ROOT.parents:
-            raise ValueError("Build output must not replace a repository")
-        if output.exists():
-            shutil.rmtree(output)
-        shutil.copytree(source, output)
-    revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
-    assets = output / "site-base"
-    assets.mkdir(parents=True, exist_ok=True)
-    has_marquee = render_marquee(output, baseurl, config.get("marquee_featured", []))
-    common_assets = ["chrome.css", "chrome.js"] + (["marquee.css", "marquee.js"] if has_marquee else [])
-    for name in common_assets:
-        shutil.copyfile(HERE / "base" / name, assets / name)
-    for name in config["pages"]:
-        page = output / name
-        text = render_metadata(page.read_text())
-        base = "../" * (len(Path(name).parts) - 1) or "./"
-        header = (HERE / "base/header.html").read_text().replace("__SITE_NAME__", escape(config["name"]))
-        header = header.replace("__SITE_NAV__", (HERE / "nav.html").read_text().strip())
-        text = replace_block(text, "HEADER", header)
-        for kind in ["FOOTER-START", "FOOTER-END"]:
-            text = replace_block(text, kind, (HERE / "base" / f"{kind.lower()}.html").read_text().strip())
-        text = text.replace("__SITE_BASE__", base).replace("__SITE_REVISION__", revision)
-        resources = f'<link rel="stylesheet" href="{base}site-base/chrome.css?v={revision}">\n<script src="{base}site-base/chrome.js?v={revision}" defer></script>'
-        if has_marquee and name == "index.html":
-            resources += f'\n<link rel="stylesheet" href="{base}site-base/marquee.css?v={revision}">\n<script src="{base}site-base/marquee.js?v={revision}" defer></script>'
-        if "<!-- ARCHIVEBOX:ASSETS -->" not in text:
-            text = text.replace("</head>", "<!-- ARCHIVEBOX:ASSETS -->\n</head>", 1)
-        text = replace_block(text, "ASSETS", resources)
-        page.write_text(text)
-    # Site build provenance is independent of all original screenshot manifests.
-    provenance_path = output / "build.json"
-    provenance = json.loads(provenance_path.read_text()) if provenance_path.exists() else {}
-    provenance.update(
-        revision=revision,
-        generatedAt=datetime.now(UTC).isoformat(),
-        site=config["url"],
-    )
-    provenance_path.write_text(json.dumps(provenance, indent=2) + "\n")
-    (output / ".nojekyll").touch()
-    verify(output, baseurl)
-    print(f"Assembled {len(config['pages'])} pages at {output} from {revision}")
-
-
-class Page(HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self.ids = set()
-        self.metadata = {}
-        self.canonical = ""
-        self.resources = []
-        self.headers = self.footers = self.canonicals = 0
-
-    def handle_starttag(self, tag, attrs):
-        attrs = dict(attrs)
-        if tag == "meta":
-            for key in ["name", "property"]:
-                if attrs.get(key):
-                    self.metadata[attrs[key]] = attrs.get("content", "")
-        if tag == "link" and attrs.get("rel") == "canonical":
-            self.canonical = attrs["href"]
-        if attrs.get("id"):
-            self.ids.add(attrs["id"])
-        classes = attrs.get("class", "").split()
-        self.headers += "abx-header" in classes
-        self.footers += "abx-footer" in classes
-        self.canonicals += tag == "link" and attrs.get("rel") == "canonical"
-        if tag in ["img", "script"] and attrs.get("src"):
-            self.resources.append(attrs["src"])
-        if tag == "link" and attrs.get("rel") in [
-            "stylesheet",
-            "icon",
-            "apple-touch-icon",
-        ]:
-            self.resources.append(attrs["href"])
-
-
-def verify(output, baseurl=""):
-    config = json.loads((HERE / "site.json").read_text())
-    output = output.resolve()
-    prefix = "/" + baseurl.strip("/") if baseurl.strip("/") else ""
-    for name in config["pages"]:
-        path = output / name
-        text = path.read_text()
-        page = Page()
-        page.feed(text)
-        if (page.headers, page.footers, page.canonicals) != (1, 1, 1):
-            raise ValueError(f"{name}: expected exactly one shared header, footer and canonical")
-        if "__SITE_" in text:
-            raise ValueError(f"{name}: unresolved site placeholder")
-        for resource in page.resources:
-            url = urlsplit(resource)
-            if url.scheme or url.netloc or not url.path:
-                continue
-            resource_path = unquote(url.path)
-            if prefix and resource_path.startswith(prefix + "/"):
-                resource_path = resource_path[len(prefix) :]
-            local = (output / resource_path.lstrip("/") if resource_path.startswith("/") else path.parent / resource_path).resolve()
-            if not local.is_relative_to(output) or not local.is_file():
-                raise ValueError(f"{name}: missing local resource {resource}")
-        for asset in ["chrome.css", "chrome.js"]:
-            if (output / "site-base" / asset).read_bytes() != (HERE / "base" / asset).read_bytes():
-                raise ValueError(f"{name}: stale common asset {asset}")
-    print(f"Verified shared layout and local resources for {len(config['pages'])} pages")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("command", choices=["render", "verify"])
-    parser.add_argument("output", type=Path)
-    parser.add_argument("--source", type=Path)
-    parser.add_argument("--baseurl", default="")
-    args = parser.parse_args()
-    if args.command == "render":
-        render(args.output, args.source, args.baseurl)
-    else:
-        verify(args.output, args.baseurl)
@@ -1,256 +0,0 @@
-"""Check the built site in real Chromium at desktop and phone widths."""
-
-import argparse
-import json
-import os
-import re
-from functools import partial
-from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
-from html.parser import HTMLParser
-from pathlib import Path
-from threading import Thread
-from urllib.parse import unquote, urljoin, urlsplit
-
-from playwright.sync_api import expect, sync_playwright
-
-HERE = Path(__file__).resolve().parent
-
-
-class Handler(SimpleHTTPRequestHandler):
-    def handle(self):
-        # Browser navigation can cancel an in-flight lazy image response.
-        try:
-            super().handle()
-        except (BrokenPipeError, ConnectionResetError):
-            pass
-
-    def log_message(self, *args):
-        pass
-
-
-class ScreenshotReferences(HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self.references = []
-        self.ids = set()
-
-    def handle_starttag(self, tag, attributes):
-        attrs = dict(attributes)
-        if attrs.get("id"):
-            self.ids.add(attrs["id"])
-        for attribute in ("src", "href"):
-            if attrs.get(attribute):
-                self.references.append(attrs[attribute])
-
-
-def verify_screenshot_references(output, canonical):
-    """Check gallery embeds on every static page, including optional translations."""
-    pages = {}
-    for path in output.rglob("*.html"):
-        page = ScreenshotReferences()
-        page.feed(path.read_text())
-        pages[path.resolve()] = page
-    for path, page in pages.items():
-        page_url = urljoin(canonical, path.relative_to(output.resolve()).as_posix())
-        for reference in page.references:
-            url = urlsplit(urljoin(page_url, reference))
-            if url.netloc != urlsplit(canonical).netloc or "/screenshots/" not in url.path:
-                continue
-            target = output / unquote(url.path).lstrip("/")
-            if target.is_dir():
-                target /= "index.html"
-            target = target.resolve()
-            assert target.is_relative_to(output.resolve()) and target.is_file(), f"{path}: missing screenshot target {reference}"
-            assert not re.match(r"^\d+-", target.name), f"{path}: screenshot filename depends on capture order: {reference}"
-            if url.fragment and target in pages:
-                assert unquote(url.fragment) in pages[target].ids, f"{path}: missing screenshot anchor {reference}"
-
-
-def verify(output, evidence):
-    config = json.loads((HERE / "site.json").read_text())
-    verify_screenshot_references(output, config["url"])
-    evidence.mkdir(parents=True, exist_ok=True)
-    server = ThreadingHTTPServer(("127.0.0.1", 0), partial(Handler, directory=str(output.resolve())))
-    thread = Thread(target=server.serve_forever, daemon=True)
-    thread.start()
-    origin = f"http://127.0.0.1:{server.server_port}"
-    try:
-        with sync_playwright() as playwright:
-            browser = playwright.chromium.launch(executable_path=os.environ.get("CHROME_BINARY"), ignore_default_args=["--hide-scrollbars"])
-            for name in config["pages"]:
-                route = name.removesuffix("index.html")
-                page = browser.new_page(reduced_motion="reduce")
-                errors = []
-                missing = []
-                page.on("pageerror", lambda error, errors=errors: errors.append(str(error)))
-                page.on(
-                    "response",
-                    lambda response, missing=missing: (
-                        missing.append(response.url) if response.url.startswith(origin) and response.status >= 400 else None
-                    ),
-                )
-                for width in [1716, 2560, 390]:
-                    page.set_viewport_size({"width": width, "height": 900})
-                    page.goto(f"{origin}/{route}", wait_until="domcontentloaded")
-                    header = page.locator(".abx-header")
-                    expect(header).to_have_count(1)
-                    expect(header).to_be_visible()
-                    expect(header).to_be_in_viewport()
-                    brand = header.locator(".abx-brand")
-                    expect(brand).to_be_visible()
-                    expect(brand).to_contain_text("ArchiveBox")
-                    expect(brand.locator(".abx-logo")).to_be_visible()
-                    expect(brand.locator(".abx-logo")).to_be_in_viewport()
-                    expect(header).to_have_css("display", "flex")
-                    expect(page.locator(".abx-footer")).to_have_count(1)
-                    expect(page.locator(".abx-footer-column")).to_have_count(3)
-                    expect(header.locator(".abx-cta")).to_be_visible()
-                    menu = header.locator(".abx-apps")
-                    menu.locator("summary").focus()
-                    page.keyboard.press("Enter")
-                    expect(menu).to_have_attribute("open", "")
-                    expect(menu.locator("a").first).to_be_visible()
-                    page.keyboard.press("Escape")
-                    expect(menu).not_to_have_attribute("open", "")
-                    menu.locator("summary").click()
-                    box = header.bounding_box()
-                    assert box
-                    page.mouse.click(box["x"] + 2, box["y"] + 5)
-                    expect(menu).not_to_have_attribute("open", "")
-                    assert page.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth"), (
-                        f"{name}: overflow at {width}"
-                    )
-                    if not route and "screenshots/index.html" in config["pages"]:
-                        strips = page.locator(".abx-marquee")
-                        assert strips.count() > 0, "Homepage must use its screenshot gallery"
-                        for strip in strips.all():
-                            strip.scroll_into_view_if_needed()
-                            bounds = strip.bounding_box()
-                            usable_width = page.evaluate("document.documentElement.clientWidth")
-                            assert bounds and abs(bounds["x"]) < 1
-                            assert abs(bounds["width"] - usable_width) < 1
-                            assert not strip.inner_text().strip(), "Screenshot strips should have no visible captions or links"
-                            cards = strip.locator(".abx-marquee-card")
-                            assert cards.count() > 0, "Screenshot strip is empty"
-                            expect(strip.locator("button")).to_have_count(0)
-                            expect(strip.locator("a:not(.abx-marquee-card)")).to_have_count(0)
-                            viewport = strip.locator(".abx-marquee-viewport")
-                            overflowing = viewport.evaluate(
-                                "node => node.scrollWidth > node.clientWidth",
-                            )
-                            if overflowing:
-                                page.mouse.move(0, 0)
-                                page.emulate_media(reduced_motion="no-preference")
-                                initial = viewport.evaluate("node => node.scrollLeft")
-                                page.wait_for_function(
-                                    "([node, initial]) => node.scrollLeft > initial",
-                                    arg=[viewport.element_handle(), initial],
-                                )
-                                viewport.hover()
-                                stopped = viewport.evaluate("node => node.scrollLeft")
-                                viewport.evaluate(
-                                    "node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
-                                )
-                                assert viewport.evaluate("node => node.scrollLeft") == stopped
-                                page.emulate_media(reduced_motion="reduce")
-                                page.mouse.move(0, 0)
-                                stopped = viewport.evaluate("node => node.scrollLeft")
-                                viewport.evaluate(
-                                    "node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
-                                )
-                                assert viewport.evaluate("node => node.scrollLeft") == stopped
-                                viewport.focus()
-                                page.keyboard.press("ArrowRight")
-                                assert viewport.evaluate("node => node.scrollWidth > node.clientWidth")
-                            else:
-                                centered = viewport.evaluate(
-                                    """node => {
-                                        const view = node.getBoundingClientRect();
-                                        const track = node.querySelector('.abx-marquee-track').getBoundingClientRect();
-                                        return Math.abs((track.left - view.left) - (view.right - track.right)) < 2;
-                                    }""",
-                                )
-                                assert centered, "Short screenshot strips must be centered"
-                            for card in cards.all():
-                                href = card.get_attribute("href")
-                                assert href and "/screenshots/" in href
-                                response = page.request.get(urljoin(page.url, href))
-                                assert response.ok, f"Broken screenshot link: {href}"
-                                anchor = urlsplit(href).fragment
-                                if anchor:
-                                    assert f'id="{anchor}"' in response.text(), f"Missing screenshot anchor: {href}"
-                    # Decode every local content image using the real browser loader,
-                    # including images in offscreen carousel tracks and lazy galleries.
-                    images = page.locator("main img").evaluate_all("nodes => [...new Set(nodes.map(image => image.src))]")
-                    local_images = [url for url in images if url.startswith(origin + "/")]
-                    page.evaluate(
-                        """async urls => {
-                        for (const url of urls) {
-                            const image = new Image();
-                            image.src = url;
-                            await image.decode();
-                            if (!image.naturalWidth) throw new Error(`Broken image: ${url}`);
-                        }
-                    }""",
-                        local_images,
-                    )
-                    page.evaluate("scrollTo({top: 0, behavior: 'instant'})")
-                    page.evaluate(
-                        "() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
-                    )
-                    page.screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}{width}.png"))
-                    page.locator(".abx-footer").screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}footer-{width}.png"))
-                    for link in header.locator(".abx-nav > a").all():
-                        url = urlsplit(link.get_attribute("href") or "")
-                        if url.scheme or url.netloc:
-                            continue
-                        response = page.request.get(urljoin(page.url, url.geturl()))
-                        assert response.ok, f"Broken navigation: {url.geturl()}"
-                        if url.fragment:
-                            body = response.text()
-                            assert f'id="{url.fragment}"' in body or f"id='{url.fragment}'" in body, (
-                                f"Missing navigation anchor: {url.geturl()}"
-                            )
-                    if route == "" and page.locator("#resources").count():
-                        page.goto(f"{origin}/#resources")
-                        expect(page.locator("#resources")).to_be_visible()
-                assert not errors, errors
-                assert not missing, missing
-                page.close()
-                context = browser.new_context(java_script_enabled=False, viewport={"width": 390, "height": 900})
-                plain = context.new_page()
-                plain.goto(f"{origin}/{route}", wait_until="domcontentloaded")
-                expect(plain.locator(".abx-header")).to_be_visible()
-                expect(plain.locator(".abx-header .abx-brand")).to_be_in_viewport()
-                expect(plain.locator(".abx-header .abx-logo")).to_be_visible()
-                plain.locator(".abx-apps summary").click()
-                expect(plain.locator(".abx-app-links a").first).to_be_visible()
-                expect(plain.locator(".abx-footer-column a").first).to_be_visible()
-                if not route and "screenshots/index.html" in config["pages"]:
-                    assert plain.locator(".abx-marquee-card").count() > 0
-                    for strip in plain.locator(".abx-marquee").all():
-                        bounds = strip.bounding_box()
-                        usable_width = plain.evaluate("document.documentElement.clientWidth")
-                        assert bounds and abs(bounds["x"]) < 1
-                        assert abs(bounds["width"] - usable_width) < 1
-                    assert plain.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth")
-                    expect(plain.locator(".abx-marquee button")).to_have_count(0)
-                    expect(plain.locator(".abx-marquee-card").first).to_be_visible()
-                context.close()
-                print(
-                    f"PASS {name}: desktop/mobile, keyboard, no-JS, images and resources",
-                    flush=True,
-                )
-            browser.close()
-    finally:
-        server.shutdown()
-        server.server_close()
-        thread.join()
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("output", type=Path)
-    parser.add_argument("--evidence", type=Path, default=Path("test-results/site"))
-    args = parser.parse_args()
-    verify(args.output, args.evidence)
@@ -1,81 +0,0 @@
-#!/usr/bin/env python3
-"""Discover every test once, batching measured short files."""
-
-import json
-import os
-from pathlib import Path
-
-
-def main() -> None:
-    root = Path(__file__).resolve().parents[2]
-    archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
-
-    if not archivebox_tests:
-        raise SystemExit("No ArchiveBox tests discovered")
-
-    # These are observed per-file pytest seconds, including fixture/startup
-    # time. Shared job setup is paid once per batch. Unknown/long files, optional
-    # dependencies and hosted-only requirements keep independent jobs.
-    durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
-    hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
-    requested = os.environ.get("TEST_FILE", "")
-    batches: list[list[str]] = []
-    batch: list[str] = []
-    batch_seconds = 0
-    for path in archivebox_tests:
-        test_path = path.relative_to(root).as_posix()
-        duration = durations.get(test_path, 60)
-        if duration >= 60 or path in hosted or path.stem == "test_auth_ldap" or test_path == requested:
-            batches.append([test_path])
-            continue
-        if batch and (len(batch) == 8 or batch_seconds + duration > 180):
-            batches.append(batch)
-            batch = []
-            batch_seconds = 0
-        batch.append(test_path)
-        batch_seconds += duration
-    if batch:
-        batches.append(batch)
-
-    matrix: list[dict[str, object]] = []
-    for paths in batches:
-        matrix.append(
-            {
-                "name": f"main/{Path(paths[0]).stem}" if len(paths) == 1 else f"batch/{Path(paths[0]).stem}",
-                "paths": paths,
-                "paths_arg": " ".join(paths),
-                "extra": "ldap" if paths == ["archivebox/tests/test_auth_ldap.py"] else "",
-                "count": len(paths),
-                "ugnas": False,
-            },
-        )
-
-    # Ordinary Linux tests can use either pool. Only genuine hosted-only
-    # requirements need a file header; filenames never enter runner policy.
-    # Keep a bounded NAS share so hosted capacity continues working in parallel.
-    # This is queued work, not a worker count: several waves keep the capped
-    # runners occupied while hosted jobs run in parallel.
-    nas_job_limit = int(os.environ.get("UGNAS_CI_MAX_JOBS", "3"))
-    # Keep long jobs early in the hosted queue. Giving the longest jobs to the
-    # CPU-capped NAS instead created a tail after hosted work had finished; use
-    # its extra slots for a bounded share of shorter work without filename rules.
-    matrix.sort(key=lambda item: sum(durations.get(path, 60) for path in item["paths"]), reverse=True)
-    eligible = [item for item in reversed(matrix) if not any(root / path in hosted for path in item["paths"])]
-    for item in eligible[:nas_job_limit]:
-        item["ugnas"] = True
-
-    discovered_paths = [path for entry in matrix for path in entry["paths"]]
-    if len(discovered_paths) != len(set(discovered_paths)):
-        raise SystemExit("Tests were not discovered exactly once")
-    if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
-        raise SystemExit("Discovered test shard coverage does not match test files")
-
-    print(
-        f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs; "
-        f"{sum(item['ugnas'] for item in matrix)} selected for ugNAS, subject to the availability check",
-    )
-    print(json.dumps(matrix, separators=(",", ":")))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,176 +0,0 @@
-{
-  "source_runs": [
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37190995942",
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37237793977"
-  ],
-  "measurement": "Unbatched files use successful per-file pytest step wall seconds from run 37237793977, including process and fixture startup, on their actual hosted or ugNAS runner. Already-batched files retain prior JUnit measurements plus 9 seconds startup. Queue and shared job setup are excluded; unknown files remain independent.",
-  "seconds": {
-    "archivebox/tests/test_api_discovery_performance.py": 28,
-    "archivebox/tests/test_api_v1_auth_browser_session.py": 17,
-    "archivebox/tests/test_api_v1_auth_check_api_token.py": 15,
-    "archivebox/tests/test_api_v1_auth_get_api_token.py": 16,
-    "archivebox/tests/test_api_v1_cli_add.py": 52,
-    "archivebox/tests/test_api_v1_cli_remove.py": 33,
-    "archivebox/tests/test_api_v1_cli_schedule.py": 29,
-    "archivebox/tests/test_api_v1_cli_search.py": 15,
-    "archivebox/tests/test_api_v1_cli_update.py": 60,
-    "archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 59,
-    "archivebox/tests/test_api_v1_core_any_id.py": 13,
-    "archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 24,
-    "archivebox/tests/test_api_v1_core_archiveresults.py": 19,
-    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 106,
-    "archivebox/tests/test_api_v1_core_snapshots.py": 17,
-    "archivebox/tests/test_api_v1_core_snapshots_rss.py": 16,
-    "archivebox/tests/test_api_v1_core_tag_tag_id.py": 15,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 16,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 16,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 16,
-    "archivebox/tests/test_api_v1_core_tags.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 16,
-    "archivebox/tests/test_api_v1_core_tags_autocomplete.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_create.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_search.py": 17,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 69,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 15,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 15,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 16,
-    "archivebox/tests/test_api_v1_crawls_crawls.py": 16,
-    "archivebox/tests/test_api_v1_machine_binaries.py": 16,
-    "archivebox/tests/test_api_v1_machine_binary_binary_id.py": 16,
-    "archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 16,
-    "archivebox/tests/test_api_v1_machine_machine_current.py": 15,
-    "archivebox/tests/test_api_v1_machine_machine_machine_id.py": 16,
-    "archivebox/tests/test_api_v1_machine_machines.py": 15,
-    "archivebox/tests/test_api_v1_personas_personas.py": 15,
-    "archivebox/tests/test_api_v1_personas_sync.py": 16,
-    "archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 27,
-    "archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 19,
-    "archivebox/tests/test_archive_result_service.py": 49,
-    "archivebox/tests/test_auth_ldap.py": 37,
-    "archivebox/tests/test_binary_service.py": 46,
-    "archivebox/tests/test_cli_add_1.py": 75,
-    "archivebox/tests/test_cli_add_2.py": 98,
-    "archivebox/tests/test_cli_add_3.py": 93,
-    "archivebox/tests/test_cli_add_4.py": 94,
-    "archivebox/tests/test_cli_archiveresult.py": 48,
-    "archivebox/tests/test_cli_background_screenshot.py": 49,
-    "archivebox/tests/test_cli_binary.py": 12,
-    "archivebox/tests/test_cli_config.py": 88,
-    "archivebox/tests/test_cli_crawl.py": 81,
-    "archivebox/tests/test_cli_extract.py": 97,
-    "archivebox/tests/test_cli_extract_input.py": 84,
-    "archivebox/tests/test_cli_help.py": 16,
-    "archivebox/tests/test_cli_init.py": 111,
-    "archivebox/tests/test_cli_install.py": 38,
-    "archivebox/tests/test_cli_list.py": 116,
-    "archivebox/tests/test_cli_machine.py": 19,
-    "archivebox/tests/test_cli_manage.py": 32,
-    "archivebox/tests/test_cli_mcp.py": 30,
-    "archivebox/tests/test_cli_oneshot.py": 15,
-    "archivebox/tests/test_cli_persona.py": 56,
-    "archivebox/tests/test_cli_piping.py": 118,
-    "archivebox/tests/test_cli_pluginmap.py": 12,
-    "archivebox/tests/test_cli_process.py": 12,
-    "archivebox/tests/test_cli_remove.py": 46,
-    "archivebox/tests/test_cli_run_1.py": 93,
-    "archivebox/tests/test_cli_run_2.py": 111,
-    "archivebox/tests/test_cli_run_3.py": 51,
-    "archivebox/tests/test_cli_run_4.py": 78,
-    "archivebox/tests/test_cli_schedule.py": 59,
-    "archivebox/tests/test_cli_search.py": 25,
-    "archivebox/tests/test_cli_server.py": 126,
-    "archivebox/tests/test_cli_shell.py": 17,
-    "archivebox/tests/test_cli_snapshot.py": 55,
-    "archivebox/tests/test_cli_status.py": 46,
-    "archivebox/tests/test_cli_tag.py": 12,
-    "archivebox/tests/test_cli_update.py": 62,
-    "archivebox/tests/test_cli_update_reindex_snapshots.py": 84,
-    "archivebox/tests/test_cli_version.py": 81,
-    "archivebox/tests/test_config_DELETE_AFTER.py": 145,
-    "archivebox/tests/test_config_MAX_limits.py": 25,
-    "archivebox/tests/test_config_ONLY_NEW.py": 15,
-    "archivebox/tests/test_config_SAVE_TITLE.py": 59,
-    "archivebox/tests/test_config_URL_filters.py": 15,
-    "archivebox/tests/test_core_config.py": 11,
-    "archivebox/tests/test_core_forms.py": 14,
-    "archivebox/tests/test_crawl_history.py": 60,
-    "archivebox/tests/test_crawl_pause.py": 11,
-    "archivebox/tests/test_crawl_runner.py": 22,
-    "archivebox/tests/test_crawl_runner_2.py": 27,
-    "archivebox/tests/test_crawl_service.py": 19,
-    "archivebox/tests/test_frozen_crawl_config.py": 26,
-    "archivebox/tests/test_hook_result_lifecycle.py": 48,
-    "archivebox/tests/test_hooks.py": 54,
-    "archivebox/tests/test_issue_1896.py": 115,
-    "archivebox/tests/test_machine_models.py": 26,
-    "archivebox/tests/test_machine_service.py": 77,
-    "archivebox/tests/test_migrations_04_to_09.py": 25,
-    "archivebox/tests/test_migrations_07_to_09.py": 86,
-    "archivebox/tests/test_migrations_08_to_09.py": 85,
-    "archivebox/tests/test_migrations_fresh.py": 55,
-    "archivebox/tests/test_misc_checks.py": 12,
-    "archivebox/tests/test_misc_monkey_patches.py": 12,
-    "archivebox/tests/test_opencode_agent_1.py": 73,
-    "archivebox/tests/test_opencode_agent_2.py": 141,
-    "archivebox/tests/test_opencode_agent_3.py": 61,
-    "archivebox/tests/test_opencode_browser_1.py": 139,
-    "archivebox/tests/test_opencode_browser_2.py": 164,
-    "archivebox/tests/test_opencode_browser_3.py": 125,
-    "archivebox/tests/test_orm_helpers.py": 14,
-    "archivebox/tests/test_permissions.py": 12,
-    "archivebox/tests/test_persona_casing.py": 45,
-    "archivebox/tests/test_persona_runtime.py": 30,
-    "archivebox/tests/test_postgres_backend.py": 79,
-    "archivebox/tests/test_process_runtime_paths.py": 20,
-    "archivebox/tests/test_process_service.py": 40,
-    "archivebox/tests/test_proof_previews.py": 24,
-    "archivebox/tests/test_recursive_crawl_1.py": 361,
-    "archivebox/tests/test_recursive_crawl_2.py": 55,
-    "archivebox/tests/test_recursive_crawl_3.py": 67,
-    "archivebox/tests/test_release_notes.py": 12,
-    "archivebox/tests/test_release_source_fingerprint.py": 13,
-    "archivebox/tests/test_release_workflow.py": 15,
-    "archivebox/tests/test_resource_admission.py": 13,
-    "archivebox/tests/test_runner_deletion.py": 91,
-    "archivebox/tests/test_runner_watch.py": 75,
-    "archivebox/tests/test_search.py": 142,
-    "archivebox/tests/test_serve_static.py": 19,
-    "archivebox/tests/test_server_automatic_url.py": 22,
-    "archivebox/tests/test_server_security_browser.py": 121,
-    "archivebox/tests/test_settings_signal_webhooks.py": 13,
-    "archivebox/tests/test_setup_script.py": 12,
-    "archivebox/tests/test_shutdown_util.py": 13,
-    "archivebox/tests/test_snapshot_filesystem_migration.py": 16,
-    "archivebox/tests/test_snapshot_output_stacks.py": 63,
-    "archivebox/tests/test_snapshot_ref_lookup.py": 15,
-    "archivebox/tests/test_snapshot_service.py": 48,
-    "archivebox/tests/test_snapshot_url_length.py": 16,
-    "archivebox/tests/test_tag_service.py": 15,
-    "archivebox/tests/test_takeover_util_1.py": 60,
-    "archivebox/tests/test_takeover_util_2.py": 62,
-    "archivebox/tests/test_takeover_util_3.py": 76,
-    "archivebox/tests/test_takeover_util_4.py": 84,
-    "archivebox/tests/test_test_harness.py": 12,
-    "archivebox/tests/test_ui_add_view.py": 26,
-    "archivebox/tests/test_ui_add_view_runtime.py": 120,
-    "archivebox/tests/test_ui_admin_apitoken.py": 16,
-    "archivebox/tests/test_ui_admin_archiveresult.py": 18,
-    "archivebox/tests/test_ui_admin_config_widget.py": 13,
-    "archivebox/tests/test_ui_admin_crawl.py": 25,
-    "archivebox/tests/test_ui_admin_links.py": 23,
-    "archivebox/tests/test_ui_admin_machine.py": 18,
-    "archivebox/tests/test_ui_admin_persona.py": 19,
-    "archivebox/tests/test_ui_admin_snapshot.py": 138,
-    "archivebox/tests/test_ui_admin_tag.py": 16,
-    "archivebox/tests/test_ui_admin_user.py": 16,
-    "archivebox/tests/test_ui_config_views.py": 20,
-    "archivebox/tests/test_ui_live_progress.py": 121,
-    "archivebox/tests/test_ui_public_snapshot.py": 72,
-    "archivebox/tests/test_ui_savepagenow.py": 31,
-    "archivebox/tests/test_urls.py": 105,
-    "archivebox/tests/test_util.py": 13,
-    "archivebox/tests/test_version_metadata.py": 18,
-    "archivebox/tests/test_zip_preview.py": 29
-  }
-}
@@ -1,103 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-    branches: [dev, main]
-  push:
-    branches: [dev, main]
-  workflow_dispatch:
-
-permissions:
-  actions: read
-  contents: read
-  packages: write
-  security-events: write
-
-concurrency:
-  group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }}
-  cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
-
-jobs:
-  prepare:
-    permissions:
-      contents: write
-    uses: ./.github/workflows/prepare-release.yml
-    secrets: inherit
-
-  lint:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Linters
-    uses: ./.github/workflows/lint.yml
-    secrets: inherit
-
-  install-cli-platform:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Install and CLI platform compatibility
-    uses: ./.github/workflows/test.yml
-    secrets: inherit
-
-  tests:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Discovered test matrix
-    uses: ./.github/workflows/test-parallel.yml
-    secrets: inherit
-
-  codeql:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: CodeQL
-    uses: ./.github/workflows/codeql.yml
-    secrets: inherit
-
-  python-artifacts:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Tested Python artifacts
-    uses: ./.github/workflows/pip.yml
-    secrets: inherit
-
-  docker-digests:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Tested Docker images
-    uses: ./.github/workflows/docker.yml
-    with:
-      push_digests: ${{ github.event_name == 'push' && (github.ref_name == 'dev' || github.ref_name == 'main') }}
-    secrets: inherit
-
-  required:
-    name: All required CI lanes
-    if: ${{ always() && needs.prepare.outputs.run_jobs == 'true' }}
-    needs:
-      - prepare
-      - lint
-      - install-cli-platform
-      - tests
-      - codeql
-      - python-artifacts
-      - docker-digests
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Verify every required lane succeeded
-        env:
-          LINT_RESULT: ${{ needs.lint.result }}
-          INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
-          TESTS_RESULT: ${{ needs.tests.result }}
-          CODEQL_RESULT: ${{ needs.codeql.result }}
-          PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
-          DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
-        run: |
-          for result in \
-            "$LINT_RESULT" \
-            "$INSTALL_CLI_PLATFORM_RESULT" \
-            "$TESTS_RESULT" \
-            "$CODEQL_RESULT" \
-            "$PYTHON_ARTIFACTS_RESULT" \
-            "$DOCKER_DIGESTS_RESULT"
-          do
-            [[ "$result" == success ]]
-          done
-          echo 'All required CI lanes and tested release artifacts succeeded.'
@@ -17,24 +17,34 @@ jobs:
      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write
      issues: write
      id-token: write
-      actions: read
+      actions: read # Required for Claude to read CI results on PRs
    steps:
      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Run Claude Code
        id: claude
-        uses: anthropics/claude-code-action@b76a0776ae74036e77cd11018083743453d7ad35 # v1
+        uses: anthropics/claude-code-action@v1
        with:
          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+
+          # This is an optional setting that allows Claude to read CI results on PRs
          additional_permissions: |
            actions: read
+
+          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
+          # prompt: 'Update the pull request description to include a summary of changes.'
+
+          # Optional: Add claude_args to customize behavior and configuration
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
          claude_args: '--allowed-tools Bash(gh pr:*)'
+
@@ -1,21 +1,42 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
 name: "CodeQL"

 on:
-  workflow_call:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
  schedule:
    - cron: '33 17 * * 6'

-concurrency:
-  group: codeql-${{ github.repository }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
  analyze:
    name: Analyze (${{ matrix.language }})
-    runs-on: ubuntu-24.04
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
    permissions:
+      # required for all workflows
      security-events: write
+
+      # required to fetch internal or private CodeQL packs
      packages: read
+
+      # only required for workflows in private repositories
      actions: read
      contents: read

@@ -25,18 +46,47 @@ jobs:
        include:
        - language: python
          build-mode: none
-        - language: javascript-typescript
-          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
    steps:
    - name: Checkout repository
-      uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      uses: actions/checkout@v4

+    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
-      uses: github/codeql-action/init@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
+      uses: github/codeql-action/init@v3
      with:
        languages: ${{ matrix.language }}
        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
+      uses: github/codeql-action/analyze@v3
      with:
        category: "/language:${{matrix.language}}"
@@ -0,0 +1,64 @@
+name: Build Debian package
+
+on:
+  workflow_dispatch:
+  push:
+
+env:
+  DEB_BUILD_OPTIONS: nocheck
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Install packaging dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y \
+            python3 python3-dev python3-pip python3-venv python3-all \
+            dh-python debhelper devscripts dput software-properties-common \
+            python3-distutils python3-setuptools python3-wheel python3-stdeb
+
+      # - name: Build Debian/Apt sdist_dsc
+      #   run: |
+      #     ./bin/build_pip.sh
+
+      # - name: Check ArchiveBox version
+      #   run: |
+      #     # must create dir needed for snaps to run as non-root on github actions
+      #     sudo mkdir -p /run/user/1001 && sudo chmod -R 777 /run/user/1001
+      #     mkdir "${{ github.workspace }}/data" && cd "${{ github.workspace }}/data"
+      #     archivebox --version
+      #     archivebox init --setup
+
+      # - name: Add some links to test
+      #   run: |
+      #     cd "${{ github.workspace }}/data"
+      #     archivebox add 'https://example.com'
+      #     archivebox status
+
+      # - name: Commit built package
+      #   run: |
+      #     cd deb_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Debian package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/debian-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: deb_dist
+
+      # - name: Push build to Launchpad PPA
+      #   run: |
+      #     debsign -k "$PGP_KEY_ID" "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
+      #     dput archivebox "deb_dist/archivebox_${VERSION}-${DEBIAN_VERSION}_source.changes"
@@ -1,96 +0,0 @@
-name: Publish website
-
-on:
-  push:
-    branches: [dev]
-  pull_request:
-  workflow_dispatch:
-  workflow_run:
-    workflows: ['Capture publicsite screenshots']
-    types: [completed]
-    branches: [dev]
-
-permissions:
-  contents: read
-  actions: read
-
-env:
-  # Repository-specific output; all publication logic below is a local copy.
-  SITE_OUTPUT: _site
-  DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-
-# One production queue covers restore, build and deploy. PRs have separate queues.
-concurrency:
-  group: pages-${{ github.event_name == 'pull_request' && github.ref || 'production' }}
-  cancel-in-progress: false
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    outputs:
-      revision: ${{ steps.source.outputs.revision }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event_name == 'pull_request' && github.ref || env.DEFAULT_BRANCH }}
-          persist-credentials: false
-      - name: Record website source revision
-        id: source
-        run: echo "revision=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
-
-      # BEGIN REPOSITORY-SPECIFIC BUILD
-      - uses: astral-sh/setup-uv@v7
-      - name: Restore latest complete screenshots
-        run: uv run --no-project python bin/restore_site_screenshots.py publicsite/screenshots
-        env:
-          GH_TOKEN: ${{ github.token }}
-      - name: Render current gallery layout without recapturing images
-        run: uv run --no-project python bin/generate_ui_screenshot_gallery.py refresh publicsite/screenshots/index.html
-      - name: Assemble website with the local base components
-        run: uv run --no-project python .github/pages/site.py render _site --source publicsite
-      # END REPOSITORY-SPECIFIC BUILD
-
-      - name: Verify website in Chromium
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/site-browser
-        run: |
-          # Resolve the same supported browser as captures; Playwright is the
-          # driver only and must not provision a separate browser/runtime.
-          ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
-          CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
-          uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
-          CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
-          export CHROME_BINARY
-          test -x "$CHROME_BINARY"
-          uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py "$SITE_OUTPUT" --evidence "$RUNNER_TEMP/site-evidence"
-      - name: Save website verification evidence
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: site-verification
-          path: ${{ runner.temp }}/site-evidence
-          if-no-files-found: warn
-          retention-days: 14
-
-      - name: Upload website
-        uses: actions/upload-pages-artifact@v3
-        with:
-          path: ${{ env.SITE_OUTPUT }}
-
-  deploy:
-    if: github.event_name != 'pull_request'
-    needs: build
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      pages: write
-      id-token: write
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    steps:
-      - uses: actions/configure-pages@v5
-      - uses: actions/deploy-pages@v4
-        id: deployment
@@ -1,143 +1,37 @@
 name: Build Docker image

 on:
-  workflow_call:
-    inputs:
-      push_digests:
-        description: Push validated per-platform images by digest for the release workflow
-        required: false
-        default: false
-        type: boolean
-      full_tests:
-        description: Run storage, CLI, and size validation in addition to release smoke checks
-        required: false
-        default: true
-        type: boolean
  workflow_dispatch:
-    inputs:
-      source_ref:
-        description: Exact 40-character commit SHA to build
-        required: true
-        type: string
-      push_digests:
-        description: Push validated per-platform images by digest for the release workflow
-        required: false
-        default: true
-        type: boolean
-      full_tests:
-        description: Run storage, CLI, and size validation in addition to release smoke checks
-        required: false
-        default: true
-        type: boolean
+  push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'
+  # pull_request:

 env:
-  DOCKERHUB_IMAGE: archivebox/archivebox
-  GHCR_IMAGE: ghcr.io/archivebox/archivebox
-
-permissions:
-  actions: read
-  contents: read
-  packages: write
+  DOCKER_IMAGE: archivebox-ci

 jobs:
-  build:
-    name: build ${{ matrix.platform }}
-    runs-on: ${{ matrix.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: linux/amd64
-            runner: ubuntu-24.04
-            artifact_name: digest-linux-amd64
-            local_tag: archivebox-ci:amd64
-          - platform: linux/arm64
-            runner: ubuntu-24.04-arm
-            artifact_name: digest-linux-arm64
-            local_tag: archivebox-ci:arm64
+  buildx:
+    runs-on: ubuntu-latest
    steps:
      - name: Checkout
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          ref: ${{ inputs.source_ref || github.sha }}
+        uses: actions/checkout@v4
+        # with:
+        #   submodules: true
+        #   fetch-depth: 1

-      - name: Verify exact source commit
-        if: github.event_name == 'workflow_dispatch'
-        shell: bash
-        env:
-          SOURCE_REF: ${{ inputs.source_ref || github.sha }}
-        run: |
-          set -Eeuo pipefail
-          [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
-          [[ "$(git rev-parse HEAD)" == "$SOURCE_REF" ]]
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Resolve Docker through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python - <<'PY'
-          import tomllib
-          from pathlib import Path
-
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          matches = [package for package in packages if package["name"] == "abxpkg"]
-          if len(matches) != 1:
-              raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}")
-          wheel = matches[0]["wheels"][0]
-          print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))
-          PY
-          )"
-          test -n "$ABXPKG_SPEC"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_SPEC=$ABXPKG_SPEC" >> "$GITHUB_ENV"
-          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-
-          docker_env="$(
-            uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries"
-          )"
-          JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$docker_env" >> "$GITHUB_ENV"
-
-          docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
-          UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv"
-          {
-            echo "DOCKER_BINARY=$docker_binary"
-            echo "JQ_BINARY=$JQ_BINARY"
-            echo "UV_BINARY=$UV_BINARY"
-          } >> "$GITHUB_ENV"
-          for binary in "$docker_binary" "$JQ_BINARY" "$UV_BINARY"; do
-            test -L "$binary"
-            test -x "$binary"
-          done
-          "$docker_binary" --version
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        id: buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+        uses: docker/setup-buildx-action@v3
        with:
+          version: latest
          install: true
+          platforms: linux/amd64,linux/arm64

      - name: Builder instance name
        run: echo ${{ steps.buildx.outputs.name }}
@@ -145,470 +39,90 @@ jobs:
      - name: Available platforms
        run: echo ${{ steps.buildx.outputs.platforms }}

-      - name: Verify published abx-dl image
-        id: abx_dl_image
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABX_DL_VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import tomllib
-          from pathlib import Path
-
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          matches = [package["version"] for package in packages if package["name"] == "abx-dl"]
-          if len(matches) != 1:
-              raise SystemExit(f"Expected one locked abx-dl package, found {len(matches)}")
-          print(matches[0])
-          PY
-          )"
-          test -n "$ABX_DL_VERSION"
-          ABX_DL_REFERENCE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
-          INSPECT_OUTPUT="$("$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_REFERENCE")"
-          printf '%s\n' "$INSPECT_OUTPUT"
-          ABX_DL_DIGEST="$(INSPECT_OUTPUT="$INSPECT_OUTPUT" "$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import os
-
-          digest = next(
-              (fields[1] for line in os.environ["INSPECT_OUTPUT"].splitlines() if (fields := line.split()) and fields[0] == "Digest:"),
-              "",
-          )
-          print(digest)
-          PY
-          )"
-          [[ "$ABX_DL_DIGEST" == sha256:* ]]
-          ABX_DL_IMAGE="${ABX_DL_REFERENCE%%@*}@${ABX_DL_DIGEST}"
-          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
+      - name: Cache Docker layers
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-

      - name: Login to Docker Hub
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        if: inputs.push_digests
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Login to GitHub Container Registry
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        if: inputs.push_digests
+        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

-      - name: Collect Docker labels
+      - name: Collect Full Release Docker tags
+        # https://github.com/docker/metadata-action
        id: docker_meta
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import tomllib
-          from pathlib import Path
+        uses: docker/metadata-action@v5
+        if: github.event_name == 'workflow_dispatch'
+        with:
+          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
+          tags: |
+              # :stable
+              type=ref,event=branch
+              # :0.7.3
+              type=semver,pattern={{version}}
+              # :0.7
+              type=semver,pattern={{major}}.{{minor}}
+              # :sha-463ea54
+              type=sha
+              # :latest
+              type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'stable') }}
+      
+      - name: Collect Non-Release Docker tags
+        # https://github.com/docker/metadata-action
+        id: docker_meta_non_release
+        uses: docker/metadata-action@v5
+        if: github.event_name != 'workflow_dispatch'
+        with:
+          images: archivebox/archivebox,ghcr.io/archivebox/archivebox
+          tags: |
+              # :stable
+              type=ref,event=branch
+              # :sha-463ea54
+              type=sha

-          print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])
-          PY
-          )"
-          test -n "$VERSION"
-
-          {
-            echo 'labels<<EOF'
-            echo "org.opencontainers.image.version=${VERSION}"
-            echo "org.opencontainers.image.revision=${{ inputs.source_ref || github.sha }}"
-            echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
-            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
-            echo 'EOF'
-            echo "version=${VERSION}"
-          } >> "$GITHUB_OUTPUT"
-
-          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
-
-      - name: Build and push digest
+      - name: Build and push
        id: docker_build
-        if: inputs.push_digests
-        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
+        uses: docker/build-push-action@v5
        with:
          context: ./
          file: ./Dockerfile
          builder: ${{ steps.buildx.outputs.name }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-          build-args: |
-            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
-          pull: true
-          platforms: ${{ matrix.platform }}
-          # Keep the released image as the one cache authority. A second GHA
-          # cache can hold gzip variants of identical layers, and BuildKit may
-          # prefer those over :dev's zstd blobs. That counter-intuitively makes
-          # a cache hit larger than the released image. :dev advances only
-          # after both architectures pass the release checks.
-          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
-          # Inline metadata reuses the released compressed blobs directly;
-          # force-compression would recompress the whole ~740 MiB image. The
-          # compressed-size gate below remains the regression guard.
-          cache-to: |
-            ${{ inputs.push_digests && 'type=inline' || '' }}
-          outputs: |
-            ${{ inputs.full_tests && format('type=oci,dest={0}/archivebox-image.tar,compression=zstd,compression-level=3', runner.temp) || '' }}
-            ${{ inputs.push_digests && format('type=image,"name={0},{1}",push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=3', env.DOCKERHUB_IMAGE, env.GHCR_IMAGE) || '' }}
-
-      - name: Build pull request image
-        id: docker_build_pr
-        if: ${{ !inputs.push_digests }}
-        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
-        with:
-          context: ./
-          file: ./Dockerfile
-          builder: ${{ steps.buildx.outputs.name }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-          build-args: |
-            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
-          pull: true
-          platforms: ${{ matrix.platform }}
-          # Test the same layer lineage users receive. Reusing :dev avoids a
-          # second cache whose compression can drift from the released image.
-          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
-          outputs: |
-            type=docker,name=${{ matrix.local_tag }}
-            type=oci,dest=${{ runner.temp }}/archivebox-image.tar
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ github.event_name == 'workflow_dispatch' ? steps.docker_meta.outputs.tags : steps.docker_meta_non_release.outputs.tags }}
+          labels: ${{ github.event_name == 'workflow_dispatch' ? steps.docker_meta.outputs.labels : steps.docker_meta_non_release.outputs.labels }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+          platforms: linux/amd64,linux/arm64

      - name: Image digest
-        if: inputs.push_digests
        run: echo ${{ steps.docker_build.outputs.digest }}

-      - name: Verify optional agent install with a read-only browser profile
-        shell: bash
-        env:
-          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: |
-          set -Eeuo pipefail
-          PROFILE_DIR="$(mktemp -d)"
-          # shellcheck disable=SC2016
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' \
-            --volume "$PROFILE_DIR:/home/archivebox/.config/chromium:ro" \
-            "$IMAGE" bash -c '
-              set -Eeuo pipefail
-              test -w "$XDG_CONFIG_HOME"
-              test ! -w "$XDG_CONFIG_HOME/chromium"
-              uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_archivebox_install; verify_archivebox_install()"
-            '
-
-      - name: Validate exact built image version and commit
-        if: inputs.full_tests
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}"
-          SHORT_SHA="${{ (inputs.source_ref || github.sha) }}"
-          SHORT_SHA="${SHORT_SHA::7}"
-
-          DATA_DIR="$(mktemp -d)"
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
-          ARCHIVEBOX_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version)"
-          printf '%s\n' "$ARCHIVEBOX_VERSION_OUTPUT"
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
-          IMAGE_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt)"
-          ARCHIVEBOX_VERSION_OUTPUT="$ARCHIVEBOX_VERSION_OUTPUT" IMAGE_VERSION_OUTPUT="$IMAGE_VERSION_OUTPUT" \
-            "$UV_BINARY" run --no-cache --no-project python - \
-            "ArchiveBox v${{ steps.docker_meta.outputs.version }}" \
-            "COMMIT_HASH=${SHORT_SHA}" \
-            "COMMIT_HASH=${GITHUB_SHA}" <<'PY'
-          import os
-          import sys
-
-          version_output = os.environ["ARCHIVEBOX_VERSION_OUTPUT"]
-          image_version_lines = os.environ["IMAGE_VERSION_OUTPUT"].splitlines()
-          if sys.argv[1] not in version_output:
-              raise SystemExit(f"Missing version marker: {sys.argv[1]}")
-          if sys.argv[2] not in version_output:
-              raise SystemExit(f"Missing short commit marker: {sys.argv[2]}")
-          if sys.argv[3] not in image_version_lines:
-              raise SystemExit(f"Missing exact commit marker: {sys.argv[3]}")
-          PY
-
-          "$DOCKER_BINARY" run --rm --user 911:911 --platform '${{ matrix.platform }}' --entrypoint /venv/bin/python3 "$IMAGE" -c \
-            'import json, pathlib, stat; cache_file = "/opt/archivebox/lib/env/derived.env"; raw = open(cache_file).read().strip(); records = json.loads(raw[len("ABXPKG_DERIVED_CACHE=\x27"):-1]); projections = [projection for record in records.values() for projection in record.get("request_exec_projections", {}).values()]; plans = [projection["validation"] for projection in projections]; fingerprints = [fingerprint for plan in plans for fingerprint in plan["fingerprint"]]; actual = lambda fingerprint: (lambda path, value: {"path": str(path.resolve(strict=False)), "size": value.st_size, "mtime_ns": value.st_mtime_ns, "mode": stat.S_IMODE(value.st_mode), "euid": value.st_uid})(pathlib.Path(fingerprint["path"]), pathlib.Path(fingerprint["path"]).stat()); mismatches = [(fingerprint, actual(fingerprint)) for fingerprint in fingerprints if fingerprint != actual(fingerprint)]; assert projections and {plan.get("euid") for plan in plans} == {911}; assert all(not record.get("script_exec_plans") for record in records.values()); assert "ABXPKG_TMP_CACHE_DIR" not in json.dumps(projections); assert not mismatches, mismatches[:3]'
-
-      - name: Validate compressed candidate size
-        if: inputs.push_digests
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          DIGEST='${{ steps.docker_build.outputs.digest }}'
-          [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
-          MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${DIGEST}")"
-          # Provenance wraps the real platform image and its attestation in an
-          # OCI index. Measure the image manifest, not the index/attestation.
-          IMAGE_DIGEST="$(MANIFEST="$MANIFEST" PLATFORM='${{ matrix.platform }}' "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); platform_os, platform_arch = os.environ["PLATFORM"].split("/")[:2]; print(next((item["digest"] for item in manifest.get("manifests", []) if item.get("platform", {}).get("os") == platform_os and item.get("platform", {}).get("architecture") == platform_arch), ""))')"
-          if [[ -n "$IMAGE_DIGEST" ]]; then
-            MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${IMAGE_DIGEST}")"
-          fi
-          COMPRESSED_SIZE="$(MANIFEST="$MANIFEST" "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); print(manifest["config"]["size"] + sum(layer["size"] for layer in manifest["layers"]))')"
-          LIMIT=$((1025 * 1024 * 1024))
-          echo "${DIGEST} compressed_size=$((COMPRESSED_SIZE / 1024 / 1024)) MiB limit=1025 MiB"
-          (( COMPRESSED_SIZE < LIMIT ))
-
-      - name: Resolve real Docker CLI test dependencies through abxpkg
-        id: mount_dependencies
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mount_env="$(
-            uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:mount_validation_binaries"
-          )"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$mount_env" >> "$GITHUB_ENV"
-
-          for binary_name in bash chmod curl exportfs mkdir mount.cifs mount.nfs smbd sudo systemctl umount; do
-            binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
-            test -L "$binary"
-            test -x "$binary"
-          done
-
-      - name: Diagnose failed mount dependency installation
-        if: failure() && steps.mount_dependencies.conclusion == 'failure'
-        shell: bash
-        run: |
-          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
-          sudo journalctl --since '-10 minutes' --no-pager \
-            -u smbd -u nmbd -u samba-ad-dc -u nfs-server
-          ps -eo pid,ppid,stat,etimes,wchan:32,comm
-
-      - name: Validate Docker mounts, ports, CLI and compose concurrently
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        env:
-          ARCHIVEBOX_IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: |
-          set -Eeuo pipefail
-          # These checks own separate collections, Compose projects and ports.
-          # Subshells keep mount cleanup traps and environment changes isolated.
-          validate_mounts() (
-            set -Eeuo pipefail
-            NFS_SOURCE="/srv/archivebox-ci-nfs"
-            SMB_SOURCE="/srv/archivebox-ci-smb"
-            NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
-            SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
-            mounted_nfs=0
-            mounted_smb=0
-
-            # Invoked by the subshell EXIT trap.
-            # shellcheck disable=SC2329
-            cleanup() {
-              local original_status=$? cleanup_status=0
-              trap - EXIT
-              if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
-                cleanup_status=1
-              fi
-              if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
-                cleanup_status=1
-              fi
-              if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
-                cleanup_status=1
-              fi
-              if [[ "$original_status" != "0" ]]; then
-                exit "$original_status"
-              fi
-              exit "$cleanup_status"
-            }
-            trap cleanup EXIT
-
-            "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-            "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
-            "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-            "$SUDO_BINARY" "$BASH_BINARY" -c \
-              "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
-            "$SUDO_BINARY" "$BASH_BINARY" -c \
-              "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
-
-            "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
-            "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
-            "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
-            "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
-            "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
-
-            "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
-            mounted_nfs=1
-            "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
-              -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
-            mounted_smb=1
-
-            NFS_TEST_DIR="$NFS_MOUNT" \
-            SMB_TEST_DIR="$SMB_MOUNT" \
-            IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
-              "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
-          )
-          validate_cli() (
-            set -Eeuo pipefail
-            "$MKDIR_BINARY" "${{ github.workspace }}/data"
-            "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
-
-            "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
-            "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
-
-            "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
-            # The argument form above exercises the complete default pipeline.
-            # Keep a real capture for stdin parsing without repeating every plugin.
-            echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add --plugins=wget,hashes
-
-            archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
-            [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
-            [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
-
-            "$DOCKER_BINARY" compose run --rm archivebox init
-            "$DOCKER_BINARY" compose up -d --wait
-            "$DOCKER_BINARY" network connect archivebox_default www-nginx
-            index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-            admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
-            [[ "$index_html" == *ArchiveBox* ]]
-            [[ "$admin_js" == *window.django* ]]
-
-            "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
-            index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-            [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
-            "$DOCKER_BINARY" compose down
-          )
-          validate_ports() (
-            IMAGE="$ARCHIVEBOX_IMAGE" "$BASH_BINARY" bin/validate_docker_ports.sh
-          )
-
-          checks=(mounts ports cli)
-          pids=()
-          validate_mounts > "$RUNNER_TEMP/docker-mounts.log" 2>&1 &
-          pids+=("$!")
-          validate_ports > "$RUNNER_TEMP/docker-ports.log" 2>&1 &
-          pids+=("$!")
-          validate_cli > "$RUNNER_TEMP/docker-cli.log" 2>&1 &
-          pids+=("$!")
-          status=0
-          for index in "${!checks[@]}"; do
-            check="${checks[$index]}"
-            echo "::group::Docker $check validation"
-            if wait "${pids[$index]}"; then
-              echo "PASS: Docker $check validation"
-            else
-              echo "::error::Docker $check validation failed"
-              status=1
-            fi
-            cat "$RUNNER_TEMP/docker-$check.log"
-            echo "::endgroup::"
-          done
-          exit "$status"
-
-      - name: Validate Docker port compatibility
-        if: ${{ !(inputs.full_tests && matrix.platform == 'linux/amd64') }}
-        shell: bash
-        env:
-          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
-
-      - name: Validate compressed image size
-        if: inputs.full_tests && !inputs.push_digests
-        env:
-          BUILD_METADATA: ${{ steps.docker_build_pr.outputs.metadata }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          LOCAL_IMAGE_ID="$("$DOCKER_BINARY" image inspect '${{ matrix.local_tag }}' --format '{{.Id}}')"
-          "$UV_BINARY" run --no-cache --no-project python - \
-            '${{ runner.temp }}/archivebox-image.tar' \
-            '${{ steps.docker_build_pr.outputs.digest }}' \
-            "$LOCAL_IMAGE_ID" \
-            "$((1025 * 1024 * 1024))" <<'PY'
-          from hashlib import sha256
-          import json
-          import os
-          import sys
-          import tarfile
-
-          archive_path, pushed_digest, local_image_id, limit_text = sys.argv[1:]
-          limit = int(limit_text)
-          metadata = json.loads(os.environ["BUILD_METADATA"])
-          if metadata["containerimage.digest"] != pushed_digest:
-              raise SystemExit("Build push transaction digest does not match the action output")
-          if not pushed_digest.startswith("sha256:") or len(pushed_digest) != 71:
-              raise SystemExit(f"Invalid pushed digest: {pushed_digest}")
-
-          with tarfile.open(archive_path) as archive:
-              def read_blob(descriptor):
-                  algorithm, digest = descriptor["digest"].split(":", 1)
-                  if algorithm != "sha256":
-                      raise SystemExit(f"Unsupported OCI digest: {descriptor['digest']}")
-                  member = archive.extractfile(f"blobs/sha256/{digest}")
-                  if member is None:
-                      raise SystemExit(f"Missing OCI blob: {descriptor['digest']}")
-                  blob = member.read()
-                  if len(blob) != descriptor["size"]:
-                      raise SystemExit(f"OCI size mismatch: {descriptor['digest']}")
-                  if sha256(blob).hexdigest() != digest:
-                      raise SystemExit(f"OCI digest mismatch: {descriptor['digest']}")
-                  return blob
-
-              index_member = archive.extractfile("index.json")
-              if index_member is None:
-                  raise SystemExit("OCI export is missing index.json")
-              index = json.load(index_member)
-
-              manifests = []
-
-              def collect(descriptor):
-                  document = json.loads(read_blob(descriptor))
-                  media_type = descriptor["mediaType"]
-                  if "image.index" in media_type or "manifest.list" in media_type:
-                      for child in document["manifests"]:
-                          collect(child)
-                  elif "image.manifest" in media_type:
-                      config_type = document["config"]["mediaType"]
-                      if "image.config" in config_type or "container.image" in config_type:
-                          manifests.append(document)
-
-              for root in index["manifests"]:
-                  collect(root)
-              if len(manifests) != 1:
-                  raise SystemExit(f"Expected one runnable image manifest, found {len(manifests)}")
-
-              manifest = manifests[0]
-              config_digest = manifest["config"]["digest"]
-              if metadata["containerimage.config.digest"] != config_digest:
-                  raise SystemExit("OCI config digest does not match the pushed build result")
-              if local_image_id != config_digest:
-                  raise SystemExit("Locally tested image does not match the OCI build result")
-              descriptors = [manifest["config"], *manifest["layers"]]
-              for descriptor in descriptors:
-                  read_blob(descriptor)
-              total = sum(descriptor["size"] for descriptor in descriptors)
-
-          print(
-              f"{pushed_digest} compressed_size={total // 1024 // 1024} MiB "
-              f"limit={limit // 1024 // 1024} MiB",
-          )
-          if total > limit:
-              raise SystemExit(f"{pushed_digest} is over the compressed size limit")
-          PY
-
-      - name: Export digest
-        if: inputs.push_digests
-        env:
-          ABX_DL_IMAGE_REF: ${{ steps.abx_dl_image.outputs.image }}
-          ARTIFACT_NAME: ${{ matrix.artifact_name }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mkdir -p /tmp/digests
-          digest="${{ steps.docker_build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-          printf '%s\n' '${{ inputs.source_ref || github.sha }}' > /tmp/digests/source-ref
-          printf '%s\n' "$ABX_DL_IMAGE_REF" > "/tmp/digests/abx-dl-image-$ARTIFACT_NAME"
-
-      - name: Upload digest
-        if: inputs.push_digests
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+      - name: Update README
+        uses: peter-evans/dockerhub-description@v4
        with:
-          name: ${{ matrix.artifact_name }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: archivebox/archivebox
+
+      # This ugly bit is necessary if you don't want your cache to grow forever
+      # until it hits GitHub's limit of 5GB.
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
@@ -1,38 +0,0 @@
-name: Documentation link checks
-
-on:
-  schedule:
-    - cron: '17 10 * * *' # Daily at 10:17 UTC, away from the busy top of the hour.
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-env:
-  PYTHONIOENCODING: utf-8
-
-jobs:
-  links:
-    name: Documentation links and anchors
-    runs-on: ubuntu-24.04
-    timeout-minutes: 5
-    permissions:
-      contents: read
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-      - name: List tracked documentation and source inputs
-        run: |
-          git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > "$RUNNER_TEMP/link-inputs.txt"
-      - uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
-        with:
-          lycheeVersion: v0.24.2
-          args: >-
-            --config .github/lychee.toml
-            --root-dir "${{ github.workspace }}"
-            --files-from "${{ runner.temp }}/link-inputs.txt"
-          fail: true
-          failIfEmpty: true
@@ -1,195 +0,0 @@
-name: Documentation tests
-
-on:
-  schedule:
-    - cron: '7 5,17 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-env:
-  PYTHONIOENCODING: utf-8
-  USE_COLOR: "False"
-  SHOW_PROGRESS: "False"
-
-jobs:
-  changes:
-    runs-on: ubuntu-24.04
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      key: ${{ steps.check.outputs.key }}
-    steps:
-      - id: check
-        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
-
-  discover:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    name: Discover documentation examples
-    runs-on: ubuntu-24.04
-    timeout-minutes: 5
-    outputs:
-      matrix: ${{ steps.inventory.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-      - name: Install ArchiveBox development dependencies
-        run: uv sync --no-cache --locked --dev
-      - name: Resolve documentation tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
-        run: |
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          uv run --no-cache --no-sync abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries"
-          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
-          mmdc_binary="$ABXPKG_LIB_DIR/env/bin/mmdc"
-          nginx_binary="$ABXPKG_LIB_DIR/env/bin/nginx"
-          test -L "$bash_binary"
-          test -L "$mmdc_binary"
-          test -L "$nginx_binary"
-          test -x "$bash_binary"
-          test -x "$mmdc_binary"
-          test -x "$nginx_binary"
-          {
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-            echo "BASH_BINARY=$bash_binary"
-          } >> "$GITHUB_ENV"
-      - id: inventory
-        name: Validate the complete authored documentation inventory
-        run: |
-          matrix="$(uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py matrix)"
-          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
-
-  examples:
-    name: docs/${{ matrix.name }}
-    needs: discover
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 15
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        with:
-          python-version: "3.13"
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-      - name: Install ArchiveBox
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        run: uv sync --no-cache --locked --dev
-      - name: Resolve documentation runtime through abxpkg
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
-        run: |
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          dependency_sources=(
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
-          )
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
-          fi
-          if [[ '${{ matrix.name }}' == ubuntu-AGENTS || '${{ matrix.name }}' == ubuntu-skills-archivebox-SKILL || '${{ matrix.name }}' == ubuntu-docs-Configuration || '${{ matrix.name }}' == ubuntu-docs-Setting-up-Authentication ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          fi
-          if [[ '${{ matrix.environment }}' == docker ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
-          elif [[ '${{ matrix.environment }}' == macos ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")
-          fi
-          uv run --no-cache --no-sync abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            "${dependency_sources[@]}"
-          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
-          python_binary="$ABXPKG_LIB_DIR/env/bin/python"
-          uv_binary="$ABXPKG_LIB_DIR/env/bin/uv"
-          test -L "$bash_binary"
-          test -x "$bash_binary"
-          test -L "$python_binary"
-          test -x "$python_binary"
-          test -L "$uv_binary"
-          test -x "$uv_binary"
-          {
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-            echo "BASH_BINARY=$bash_binary"
-            echo "PYTHON_BINARY=$python_binary"
-            echo "UV_BINARY=$uv_binary"
-          } >> "$GITHUB_ENV"
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            sudo_binary="$ABXPKG_LIB_DIR/env/bin/sudo"
-            test -L "$sudo_binary"
-            test -x "$sudo_binary"
-            echo "SUDO_BINARY=$sudo_binary" >> "$GITHUB_ENV"
-          fi
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-          fi
-          if [[ '${{ matrix.environment }}' == docker ]]; then
-            docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
-            test -L "$docker_binary"
-            test -x "$docker_binary"
-            echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV"
-          elif [[ '${{ matrix.environment }}' == macos ]]; then
-            test -L "$ABXPKG_LIB_DIR/env/bin/brew"
-            test -x "$ABXPKG_LIB_DIR/env/bin/brew"
-          fi
-      - name: Build the exact checkout's documentation image
-        if: ${{ matrix.environment == 'docker' }}
-        run: |
-          ABX_DL_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(dep.removeprefix("abx-dl==") for dep in tomllib.load(open("pyproject.toml", "rb"))["project"]["dependencies"] if dep.startswith("abx-dl==")))')"
-          "$DOCKER_BINARY" build --build-arg "ABX_DL_IMAGE=archivebox/abx-dl:${ABX_DL_VERSION}" --tag archivebox/archivebox:dev .
-      - name: Run the Ubuntu, Docker, or macOS documentation lane
-        if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
-        run: uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
-      - name: Run the root documentation lane
-        if: ${{ matrix.environment == 'root' }}
-        run: |
-          "$SUDO_BINARY" \
-            --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
-            "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
-      - name: Diagnose failed Linux package installation
-        if: failure() && runner.os == 'Linux'
-        run: |
-          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
-          apt-config dump Acquire::Retries Acquire::http::Timeout Acquire::https::Timeout
-          ps -eo pid,ppid,stat,etimes,wchan:32,comm
-
-  complete:
-    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
-    needs: [changes, discover, examples]
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - run: echo 'Scheduled work completed successfully.'
@@ -0,0 +1,59 @@
+name: Duplicate Issue Detection
+
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  check-duplicates:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Install opencode
+        run: curl -fsSL https://opencode.ai/install | bash
+
+      - name: Check for duplicate issues
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENCODE_PERMISSION: |
+            {
+              "bash": {
+                "gh issue*": "allow",
+                "gh pr*": "allow",
+                "*": "deny"
+              },
+              "webfetch": "allow"
+            }
+        run: |
+          opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created:
+
+          Issue number:
+          ${{ github.event.issue.number }}
+
+          Lookup this issue and search through existing issues and PRs (excluding #${{ github.event.issue.number }}) in this repository to find any potential duplicates of this new issue.
+          Consider:
+          1. Similar titles or descriptions
+          2. Same error messages or symptoms
+          3. Related functionality or components
+          4. Similar feature requests
+
+          If you find any potential duplicates, please comment on the new issue with:
+          - A brief explanation of why it might be a duplicate
+          - Links to the potentially duplicate issues or PRs
+          - A suggestion to check those issues first
+
+          Use this format for the comment:
+          'This issue might be a duplicate of existing issues. Please check:
+          - #[issue_number]: [brief description of similarity]
+
+          Feel free to ignore if none of these address your specific case.'
+
+          If no clear duplicates are found, do not comment."
@@ -0,0 +1,64 @@
+# Simple workflow for deploying static content to GitHub Pages
+name: Deploy static content to Pages
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["dev"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Single deploy job since we're just deploying
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Copy README.md into place
+        run: |
+          rm -f ./website/README.md
+          cp ./README.md ./website/README.md
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+
+      # - name: Checkout
+      #   uses: actions/checkout@v4
+      # - name: Setup Pages
+      #   uses: actions/configure-pages@v5
+      # - name: Upload artifact
+      #   uses: actions/upload-pages-artifact@v3
+      #   with:
+      #     # Upload entire repository
+      #     path: './website'
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
@@ -0,0 +1,51 @@
+name: Build Homebrew package
+
+on:
+  workflow_dispatch:
+  push:
+
+
+jobs:
+  build:
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      # TODO: modify archivebox.rb to update src url, hashes, and dependencies
+
+      - name: Build Homebrew Bottle
+        run: |
+          pip3 install --upgrade pip setuptools wheel
+          cd brew_dist/
+          brew install --build-bottle ./archivebox.rb
+          # brew bottle archivebox
+          archivebox version
+
+      - name: Add some links to test
+        run: |
+          mkdir data && cd data
+          archivebox init --setup
+          archivebox add 'https://example.com'
+          archivebox version
+          archivebox status
+
+      # - name: Commit built package
+      #   run: |
+      #     cd brew_dist/
+      #     git config --local user.email "action@github.com"
+      #     git config --local user.name "GitHub Action"
+      #     git commit -m "Homebrew package autobuild" -a
+      
+      # - name: Push build to Github
+      #   uses: ad-m/github-push-action@master
+      #   with:
+      #     github_token: ${{ secrets.GITHUB_TOKEN }}
+      #     repository: ArchiveBox/homebrew-archivebox
+      #     branch: ${{ github.ref }}
+      #     directory: brew_dist
+
+      # TODO: push bottle homebrew core PR with latest changes
@@ -0,0 +1,58 @@
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Build GitHub Pages website
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["dev"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Copy README.md into place
+        run: |
+          rm ./website/README.md
+          cp ./README.md ./website/README.md
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2
@@ -1,38 +1,35 @@
 name: Run linters

 on:
-  workflow_call:
+  workflow_dispatch:
+  push:

 env:
-  UV_NO_SOURCES: "1"
+  MAX_LINE_LENGTH: 110

 jobs:
  lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v2
        with:
          submodules: true
          fetch-depth: 1

      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/setup-python@v1
        with:
-          python-version: "3.13"
+          python-version: 3.9
          architecture: x64

-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+      - name: Install flake8
+        run: |
+          pip install flake8

-      - name: Verify checked dependency lock
-        run: uv lock --no-cache --check
-
-      - name: Install dependencies with uv
-        run: uv sync --no-cache --locked --dev --extra sonic --extra debug
-
-      - name: Run prek
-        run: uv run --no-cache --no-sync --no-sources prek run --all-files
+      - name: Lint with flake8
+        run: |
+          cd archivebox
+          # one pass for show-stopper syntax errors or undefined names
+          flake8 . --count --show-source --statistics
+          # one pass for small stylistic things
+          flake8 . --count --max-line-length="$MAX_LINE_LENGTH" --statistics
@@ -1,48 +0,0 @@
-name: macOS compatibility
-
-on:
-  schedule:
-    - cron: '51 2,10,18 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-jobs:
-  changes:
-    runs-on: ubuntu-24.04
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      key: ${{ steps.check.outputs.key }}
-    steps:
-      - id: check
-        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
-
-  install-cli:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    uses: ./.github/workflows/test.yml
-    with:
-      platform: macos
-    secrets: inherit
-
-  python-artifacts:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    uses: ./.github/workflows/pip.yml
-    with:
-      platforms: '["macos-15"]'
-    secrets: inherit
-
-  complete:
-    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
-    needs: [changes, install-cli, python-artifacts]
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - run: echo 'Scheduled macOS coverage completed successfully.'
@@ -1,248 +1,65 @@
 name: Build Pip package

 on:
-  workflow_call:
-    inputs:
-      platforms:
-        description: Runner matrix for package compatibility
-        required: false
-        default: '["ubuntu-24.04"]'
-        type: string
-      full_tests:
-        description: Run the full package compatibility matrix instead of one release smoke check
-        required: false
-        default: true
-        type: boolean
+  workflow_dispatch:
+  push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'

 env:
-  UV_VERSION: "0.11.3"
+  PYTHON_VERSION: "3.13"

 jobs:
  build:
-    name: build distributions
+    permissions:
+      id-token: write
+
    runs-on: ubuntu-24.04
    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4

-      - name: Set up Python 3.13
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
        with:
-          python-version: "3.13"
+          enable-cache: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
          architecture: x64

-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+      - name: APT install archivebox dev + run dependencies
+        uses: awalsh128/cache-apt-pkgs-action@latest
        with:
-          version: ${{ env.UV_VERSION }}
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0

-      - name: Build every distribution
+      - name: UV install archivebox dev + run sub-dependencies
+        run: uv sync --frozen --all-extras --no-install-project --no-install-workspace
+
+      - name: UV build archivebox and archivebox/pkgs/* packages
        run: |
-          set -Eeuo pipefail
-          uv build --no-cache --all-packages --no-sources --out-dir dist --clear
+          uv build --all

-          shopt -s nullglob
-          artifacts=(dist/*)
-          wheels=(dist/archivebox-*.whl)
-          sdists=(dist/archivebox-*.tar.gz)
-          [[ "${#artifacts[@]}" -gt 0 ]]
-          [[ "${#wheels[@]}" -eq 1 ]]
-          [[ "${#sdists[@]}" -eq 1 ]]
-          printf '%s\n' "$GITHUB_SHA" > dist/COMMIT_SHA
-          uv run --no-cache --no-project python - <<'PY'
-          import hashlib
-          from pathlib import Path
+      - name: Publish new package wheels and sdists to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # ignore when publish to PyPI fails due to duplicate tag
+        continue-on-error: true
+        with:
+          password: ${{ secrets.PYPI_PAT_SECRET }}

-          root = Path("dist")
-          wheels = list(root.glob("archivebox-*.whl"))
-          sdists = list(root.glob("archivebox-*.tar.gz"))
-          if len(wheels) != 1 or len(sdists) != 1:
-              raise SystemExit("Expected exactly one ArchiveBox wheel and one sdist")
-          artifacts = [root / "COMMIT_SHA", wheels[0], sdists[0]]
-          checksum_lines = []
-          for artifact in artifacts:
-              digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
-              line = f"{digest}  {artifact.name}"
-              checksum_lines.append(line)
-              print(line)
-          Path("dist/SHA256SUMS").write_text("\n".join(checksum_lines) + "\n")
-          PY
+      - name: UV install archivebox and archivebox/pkgs/* locally for tests
+        run: uv sync --frozen --all-extras

-      - name: Release wheel import and CLI smoke
-        if: ${{ !inputs.full_tests }}
-        shell: bash
+      - name: UV run archivebox init + archivebox version
        run: |
-          set -Eeuo pipefail
-          shopt -s nullglob
-          wheels=(dist/archivebox-*.whl)
-          [[ "${#wheels[@]}" -eq 1 ]]
-          uv sync --locked --no-cache --no-sources --no-dev --no-install-project
-          uv pip install --no-cache --no-deps --python .venv/bin/python "${wheels[0]}"
-          unset PYTHONPATH
-          cd "$(mktemp -d)"
-          VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project python -c 'import archivebox'
-          VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project archivebox version
+          mkdir -p data && cd data
+          uv run archivebox init \
+          && uv run archivebox version
+          # && uv run archivebox add 'https://example.com' \
+          # && uv run archivebox status \
+          # || (echo "UV Failed to run archivebox!" && exit 1)

-      - name: Upload every distribution
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: python-distributions
-          path: dist/*
-          if-no-files-found: error
-          retention-days: 7
-
-  install-smoke:
-    name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }}
-    if: inputs.full_tests
-    needs: build
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ${{ fromJSON(inputs.platforms) }}
-        python: ["3.13", "3.14"]
-        artifact: [wheel, sdist]
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: ${{ env.UV_VERSION }}
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Download distributions
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: python-distributions
-          path: ${{ runner.temp }}/python-distributions
-
-      - name: Install and smoke-test the built ${{ matrix.artifact }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          shopt -s nullglob
-
-          case "${{ matrix.artifact }}" in
-            wheel) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.whl) ;;
-            sdist) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.tar.gz) ;;
-            *) exit 2 ;;
-          esac
-          [[ "${#candidates[@]}" -eq 1 ]]
-          artifact="${candidates[0]}"
-
-          smoke_root="$(mktemp -d "$RUNNER_TEMP/archivebox-package-smoke.XXXXXX")"
-          trap 'rm -rf "$smoke_root"' EXIT
-          smoke_env="$smoke_root/env"
-          data_dir="$smoke_root/data"
-          mkdir -p "$data_dir"
-
-          dependency_artifacts=()
-          while IFS= read -r dependency_artifact; do
-            dependency_artifacts+=("$dependency_artifact")
-          done < <(uv run --no-cache --no-project python - <<'PY'
-          import tomllib
-
-          locked = {
-              package["name"]: package
-              for package in tomllib.load(open("uv.lock", "rb"))["package"]
-          }
-          for package_name in ("abx-dl", "abx-plugins", "abxpkg", "abxbus"):
-              package = locked[package_name]
-              wheels = package.get("wheels") or []
-              if len(wheels) != 1:
-                  raise SystemExit(f"Expected exactly one locked wheel for {package_name}")
-              print(wheels[0]["url"])
-          PY
-          )
-
-          uv venv --python "${{ matrix.python }}" "$smoke_env"
-          uv pip install --no-cache \
-            --python "$smoke_env" \
-            "${dependency_artifacts[@]}" \
-            "$artifact"
-
-          unset PYTHONPATH
-          cd "$data_dir"
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync python - <<'PY'
-          import os
-          from pathlib import Path
-
-          import archivebox
-
-          installed_path = Path(archivebox.__file__).resolve()
-          environment_path = Path(os.environ["VIRTUAL_ENV"]).resolve()
-          workspace_path = Path(os.environ["GITHUB_WORKSPACE"]).resolve()
-          assert installed_path.is_relative_to(environment_path), (installed_path, environment_path)
-          assert not installed_path.is_relative_to(workspace_path), (installed_path, workspace_path)
-          print(installed_path)
-          PY
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox version
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox init
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox status
-
-          if [[ "${{ runner.os }}" == "Linux" ]]; then
-            root_tool_dir="/root/archivebox-package-smoke-${{ matrix.artifact }}-${{ matrix.python }}"
-            uv_binary="$(command -v uv)"
-            sudo rm -rf "$root_tool_dir"
-            sudo mkdir -p "$root_tool_dir/data"
-
-            root_install_args=()
-            for dependency_artifact in "${dependency_artifacts[@]}"; do
-              root_install_args+=(--with "$dependency_artifact")
-            done
-            sudo UV_TOOL_DIR="$root_tool_dir/tools" UV_TOOL_BIN_DIR="$root_tool_dir/bin" \
-              "$uv_binary" tool install --no-cache --python "${{ matrix.python }}" \
-              "${root_install_args[@]}" "$artifact"
-
-            sudo useradd --system --create-home --home-dir /var/lib/archivebox --shell /bin/bash archivebox 2>/dev/null || true
-            sudo rm -rf /var/lib/archivebox
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data' && archivebox init"
-            sudo stat -c %U /var/lib/archivebox | grep -qx archivebox
-            sudo stat -c %U "$root_tool_dir/data" | grep -qx archivebox
-            if sudo find "$root_tool_dir/data" -xdev -user root -print -quit | grep -q .; then
-              echo "Root-owned files remain in $root_tool_dir/data" >&2
-              exit 1
-            fi
-
-            sudo mkdir "$root_tool_dir/external"
-            sudo ln -s "$root_tool_dir/external" "$root_tool_dir/data/external-link"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data' && archivebox status"
-            sudo stat -c %U "$root_tool_dir/external" | grep -qx root
-
-            sudo mkdir "$root_tool_dir/data-universal-init"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-universal-init' && archivebox status --init"
-            sudo stat -c %U "$root_tool_dir/data-universal-init" | grep -qx archivebox
-
-            sudo mkdir "$root_tool_dir/data-install"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-install' && archivebox install --dry-run"
-            sudo stat -c %U "$root_tool_dir/data-install" | grep -qx archivebox
-
-            sudo useradd --create-home archivebox-smoke-user
-            smoke_user_group="$(id -gn archivebox-smoke-user)"
-            sudo mkdir "$root_tool_dir/data-mixed-owner"
-            sudo chown archivebox-smoke-user:root "$root_tool_dir/data-mixed-owner"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-mixed-owner' && archivebox init"
-            sudo stat -c %U:%G "$root_tool_dir/data-mixed-owner" | grep -qx "archivebox-smoke-user:$smoke_user_group"
-            if sudo find "$root_tool_dir/data-mixed-owner" -xdev -user root -print -quit | grep -q .; then
-              echo "Root-owned files remain in $root_tool_dir/data-mixed-owner" >&2
-              exit 1
-            fi
-            sudo rm -rf "$root_tool_dir"
-          fi
@@ -1,55 +0,0 @@
-name: Prepare release version
-
-on:
-  workflow_call:
-    outputs:
-      run_jobs:
-        description: Whether this revision owns its release version and should run expensive jobs
-        value: ${{ jobs.prepare.outputs.run_jobs }}
-
-permissions:
-  contents: write
-
-jobs:
-  prepare:
-    name: Prepare the version before starting expensive jobs
-    permissions:
-      contents: write
-    runs-on: ubuntu-24.04
-    outputs:
-      run_jobs: ${{ github.event_name != 'push' || steps.channel.outputs.stable == 'true' || steps.version.outputs.action == 'ready' || steps.version.outputs.action == 'reserved' }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          token: ${{ github.event_name == 'push' && secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - name: Keep synchronized stable branches on the released version
-        id: channel
-        if: github.event_name == 'push'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
-          git fetch --no-tags origin main
-          MAIN_SHA="$(git rev-parse origin/main)"
-          # Stable reconciliation may preserve dev history with a merge commit.
-          # Its tree still matches main; bumping it would undo synchronization
-          # before the stable publisher can advance the Docker :dev tag.
-          if [[ "$GITHUB_REF_NAME" == dev && "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
-             git merge-base --is-ancestor "$MAIN_SHA" "$GITHUB_SHA" &&
-             git diff --quiet "$MAIN_SHA" "$GITHUB_SHA"; then
-            echo 'stable=true' >> "$GITHUB_OUTPUT"
-          else
-            echo 'stable=false' >> "$GITHUB_OUTPUT"
-          fi
-
-      - id: version
-        if: github.event_name == 'push' && steps.channel.outputs.stable != 'true'
-        uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
-        with:
-          package: archivebox
-          branch: ${{ github.ref_name }}
-          tag-prefix: v
-          version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}
@@ -1,595 +0,0 @@
-name: Release
-
-on:
-  workflow_run:
-    workflows: [CI]
-    types: [completed]
-    branches: [dev, main]
-  workflow_dispatch:
-    inputs:
-      release_branch:
-        description: Branch being promoted
-        required: true
-        type: choice
-        options: [main, dev]
-      release_sha:
-        description: Exact tested commit to release
-        required: true
-        type: string
-      ci_run_id:
-        description: Successful CI run containing the tested Python artifacts
-        required: true
-        type: string
-      docker_digest_run_id:
-        description: Manual Docker digest build run containing the tested image digests
-        required: true
-        type: string
-
-permissions: {}
-
-concurrency:
-  group: release-${{ github.repository }}-${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
-  cancel-in-progress: false
-
-env:
-  DOCKERHUB_IMAGE: archivebox/archivebox
-  GHCR_IMAGE: ghcr.io/archivebox/archivebox
-  RELEASE_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
-  RELEASE_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.release_sha || github.event.workflow_run.head_sha }}
-
-jobs:
-  candidate:
-    if: >-
-      github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.event == 'push' &&
-      (github.event.workflow_run.head_branch == 'dev' || github.event.workflow_run.head_branch == 'main') ||
-      (github.event_name == 'workflow_dispatch' && inputs.release_branch == 'main')
-    runs-on: ubuntu-24.04
-    permissions:
-      actions: read
-      contents: read
-    outputs:
-      ready: ${{ steps.verified.outputs.ready }}
-      artifact_run_id: ${{ steps.verified.outputs.artifact_run_id }}
-      digest_run_id: ${{ steps.verified.outputs.digest_run_id }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          ref: ${{ env.RELEASE_SHA }}
-      - id: owner
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          BRANCH_SHA="$(git ls-remote origin "refs/heads/$RELEASE_BRANCH" | cut -f1)"
-          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && "$BRANCH_SHA" != "$RELEASE_SHA" ]]; then
-            echo 'ready=false' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
-              echo "Refusing to publish non-stable version $VERSION as a main release" >&2
-              exit 1
-            }
-            git fetch --no-tags origin main
-            git merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
-              echo "Refusing to publish $RELEASE_SHA: it is not on main" >&2
-              exit 1
-            }
-            TAG_TARGET="$(git ls-remote origin "refs/tags/v$VERSION" | cut -f1)"
-            if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && -n "$TAG_TARGET" && "$TAG_TARGET" != "$RELEASE_SHA" ]]; then
-              echo 'ready=false' >> "$GITHUB_OUTPUT"
-              exit 0
-            fi
-            echo 'ready=true' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          # A stable release fast-forwards dev to main. Only main publishes it;
-          # the resulting dev workflow is not a new release candidate.
-          if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]]; then
-            echo 'ready=false' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          TARGET="$(git ls-remote origin "refs/tags/release-candidate/$VERSION" | awk '{print $1}')"
-          [[ "$TARGET" == "$RELEASE_SHA" ]] && echo 'ready=true' >> "$GITHUB_OUTPUT" || echo 'ready=false' >> "$GITHUB_OUTPUT"
-
-      - name: Require successful full CI for this commit
-        id: verified
-        if: steps.owner.outputs.ready == 'true'
-        env:
-          GH_TOKEN: ${{ github.token }}
-          MANUAL_CI_RUN_ID: ${{ inputs.ci_run_id }}
-          MANUAL_DIGEST_RUN_ID: ${{ inputs.docker_digest_run_id }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then
-            CI_RUN_ID="$MANUAL_CI_RUN_ID"
-            DIGEST_RUN_ID="$MANUAL_DIGEST_RUN_ID"
-            CI_STATUS="$(gh run view "$CI_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status,conclusion,headSha,event,name \
-              --jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
-            [[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
-          else
-            CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
-              --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
-              --json databaseId,status,conclusion \
-              --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
-            [[ -n "$CI_RUN_ID" ]] || exit 0
-            DIGEST_RUN_ID="$CI_RUN_ID"
-          fi
-          ARTIFACT_RUN_ID="$CI_RUN_ID"
-          {
-            echo 'ready=true'
-            echo "artifact_run_id=$ARTIFACT_RUN_ID"
-            echo "digest_run_id=$DIGEST_RUN_ID"
-          } >> "$GITHUB_OUTPUT"
-
-  python-release:
-    name: Publish tested Python artifacts
-    needs: candidate
-    if: needs.candidate.outputs.ready == 'true'
-    runs-on: ubuntu-24.04
-    env:
-      CI_RUN_ID: ${{ needs.candidate.outputs.artifact_run_id }}
-      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
-    environment: pypi
-    permissions:
-      actions: read
-      contents: write
-      deployments: read
-      id-token: write
-    outputs:
-      version: ${{ steps.version.outputs.version }}
-      tag: ${{ steps.version.outputs.tag }}
-      release_ready: ${{ steps.release-state.outputs.ready }}
-    steps:
-      - name: Checkout the tested commit
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          ref: ${{ env.RELEASE_SHA }}
-          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      # This precedes every stable upload, including workflow_dispatch. A green
-      # build is not evidence that our automatically updated test servers ran it.
-      - name: Download tested Docker base-image metadata
-        if: env.RELEASE_BRANCH == 'main'
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          path: ${{ runner.temp }}/tested-docker-artifacts
-          pattern: digest-*
-          merge-multiple: true
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
-
-      - name: Require matching acceptance on Cabbage
-        if: env.RELEASE_BRANCH == 'main'
-        env:
-          GH_TOKEN: ${{ github.token }}
-          STAGING_DOCKER_ARTIFACT_DIR: ${{ runner.temp }}/tested-docker-artifacts
-        run: uv run --no-project python bin/verify-staging-release.py
-
-      - name: Resolve release binaries through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" >/dev/null
-          for name in uv gh git jq curl; do
-            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
-            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
-          done
-          {
-            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
-            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
-            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-            echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
-          } >> "$GITHUB_ENV"
-
-      - name: Download exact tested distributions
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: python-distributions
-          path: ${{ runner.temp }}/python-distributions
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.CI_RUN_ID }}
-
-      - name: Record release version
-        id: version
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$($UV_BINARY run --no-cache --no-project python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
-          echo "tag=v$VERSION" >> "$GITHUB_OUTPUT"
-
-      - name: Validate release branch and version
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION='${{ steps.version.outputs.version }}'
-          case "$RELEASE_BRANCH" in
-            main)
-              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
-                echo "Refusing to publish non-stable version $VERSION from main" >&2
-                exit 1
-              }
-              ;;
-            dev)
-              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]] || {
-                echo "Refusing to publish ArchiveBox $VERSION from dev; only archivebox:main may publish non-prerelease versions." >&2
-                exit 1
-              }
-              ;;
-            *)
-              echo "Refusing to publish ArchiveBox $VERSION from unsupported branch $RELEASE_BRANCH; only archivebox:dev and archivebox:main may publish." >&2
-              exit 1
-              ;;
-          esac
-
-      - name: Publish the exact tested distributions
-        env:
-          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-          RELEASE_DISTRIBUTIONS_DIR: ${{ runner.temp }}/python-distributions
-        run: ./bin/release.sh
-
-      - name: Confirm this tested commit owns the release tag
-        id: release-state
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          TARGET=""
-          while read -r sha _; do
-            TARGET="$sha"
-          done < <("$GIT_BINARY" ls-remote origin "refs/tags/${{ steps.version.outputs.tag }}")
-          if [[ "$TARGET" == "$RELEASE_SHA" ]]; then
-            echo "ready=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "ready=false" >> "$GITHUB_OUTPUT"
-          fi
-
-  docker-release:
-    name: Publish tested multiarch image
-    if: >-
-      (github.event_name == 'workflow_dispatch') ||
-      (github.event.workflow_run.conclusion == 'success' &&
-       github.event.workflow_run.event == 'push')
-    needs: [candidate, python-release]
-    runs-on: ubuntu-24.04
-    env:
-      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
-    permissions:
-      actions: read
-      contents: write
-      packages: write
-    steps:
-      - name: Checkout the tested commit
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          ref: ${{ env.RELEASE_SHA }}
-          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - name: Reconcile dev to the tested stable release
-        if: env.RELEASE_BRANCH == 'main' && needs.python-release.outputs.release_ready == 'true'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          git fetch --no-tags origin main dev
-          MAIN_SHA="$(git rev-parse origin/main)"
-          DEV_SHA="$(git rev-parse origin/dev)"
-          if [[ "$MAIN_SHA" != "$RELEASE_SHA" || "$DEV_SHA" == "$RELEASE_SHA" ]]; then
-            exit 0
-          fi
-          push_dev() {
-            if ! git push origin "$1:refs/heads/dev"; then
-              CURRENT_DEV="$(git ls-remote origin refs/heads/dev | cut -f1)"
-              if [[ "$CURRENT_DEV" == "$DEV_SHA" ]]; then
-                echo "Failed to push dev although it still points to $DEV_SHA" >&2
-                exit 1
-              fi
-              echo "Leaving dev at $CURRENT_DEV: it advanced during release reconciliation"
-            fi
-          }
-          if git merge-base --is-ancestor "$DEV_SHA" "$RELEASE_SHA"; then
-            push_dev "$RELEASE_SHA"
-            exit 0
-          else
-            STATUS=$?
-            [[ "$STATUS" -eq 1 ]] || exit "$STATUS"
-          fi
-          BASE_SHA="$(git merge-base "$MAIN_SHA" "$DEV_SHA")"
-          if git diff --quiet "$BASE_SHA" "$DEV_SHA" -- . ':!pyproject.toml' ':!uv.lock' ':!etc/package.json'; then
-            :
-          else
-            STATUS=$?
-            [[ "$STATUS" -eq 1 ]] || exit "$STATUS"
-            echo "Leaving dev at $DEV_SHA: it contains changes beyond release metadata"
-            exit 0
-          fi
-          normalized_metadata() {
-            case "$2" in
-              pyproject.toml)
-                git show "$1:$2" | awk '
-                  /^\[/ {section = $0}
-                  section == "[project]" && /^version = "[^"]+"$/ {print "version = \"__VERSION__\""; next}
-                  section == "[tool.bumpver]" && /^current_version = "[^"]+"$/ {print "current_version = \"__VERSION__\""; next}
-                  {print}
-                ' ;;
-              uv.lock)
-                git show "$1:$2" | awk 'previous == "name = \"archivebox\"" && /^version = / {print "version = \"__VERSION__\""; previous = $0; next} {print; previous = $0}' ;;
-              etc/package.json)
-                git show "$1:$2" | sed -E 's/^(  "version": )"[^"]+"(,?)$/\1"__VERSION__"\2/' ;;
-            esac
-          }
-          for path in pyproject.toml uv.lock etc/package.json; do
-            BASE_METADATA="$(normalized_metadata "$BASE_SHA" "$path")"
-            DEV_METADATA="$(normalized_metadata "$DEV_SHA" "$path")"
-            if [[ "$BASE_METADATA" != "$DEV_METADATA" ]]; then
-              echo "Leaving dev at $DEV_SHA: $path contains changes beyond release version metadata"
-              exit 0
-            fi
-          done
-          # A concurrent release bot may bump only dev's own version metadata.
-          # Keep that commit as a parent while preserving the exact tested stable tree.
-          git switch --detach "$MAIN_SHA"
-          git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \
-            merge -s ours --no-ff -m "Merge tested stable release into dev" "$DEV_SHA"
-          MERGE_SHA="$(git rev-parse HEAD)"
-          push_dev "$MERGE_SHA"
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      - name: Resolve release and Docker binaries through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
-          for name in uv gh git jq docker; do
-            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
-            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
-          done
-          {
-            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
-            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
-            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-            echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
-          } >> "$GITHUB_ENV"
-
-      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
-        with:
-          # These images were already built and tested; only publish manifests.
-          driver: docker
-
-      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ github.token }}
-
-      - name: Download exact tested image digests
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          path: /tmp/digests
-          pattern: digest-*
-          merge-multiple: true
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
-
-      - name: Verify exact successful digest build run
-        shell: bash
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          set -Eeuo pipefail
-          RUN_STATUS="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status --jq '.status')"
-          RUN_CONCLUSION="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json conclusion --jq '.conclusion')"
-          [[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]]
-          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
-            RUN_SHA="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha')"
-            [[ "$RUN_SHA" == "$RELEASE_SHA" ]]
-          fi
-
-      - name: Verify digest set and collect tags
-        id: docker_meta
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          [[ "${#DIGESTS[@]}" -eq 2 ]]
-          for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
-          [[ "$(< /tmp/digests/source-ref)" == "$RELEASE_SHA" ]]
-
-          VERSION='${{ needs.python-release.outputs.version }}'
-          SHORT_SHA="${RELEASE_SHA::12}"
-          TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}^{}")"
-          TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
-          if [[ -z "$TAG_TARGET" ]]; then
-            TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
-            TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
-          fi
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
-              echo "Refusing to tag :latest with prerelease $VERSION" >&2
-              exit 1
-            }
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] || {
-              echo "Refusing to tag :latest: v$VERSION does not point to $RELEASE_SHA" >&2
-              exit 1
-            }
-            $GIT_BINARY fetch --no-tags origin main
-            $GIT_BINARY merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
-              echo "Refusing to tag :latest: $RELEASE_SHA is not on main" >&2
-              exit 1
-            }
-            MAIN_VERSION="$($GIT_BINARY show origin/main:pyproject.toml | sed -nE 's/^version = "([^"]+)".*/\1/p' | head -n 1)"
-            [[ "$MAIN_VERSION" == "$VERSION" ]] || {
-              echo "Refusing to tag :latest: main is now $MAIN_VERSION, not $VERSION" >&2
-              exit 1
-            }
-          fi
-          SYNC_DEV=false
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            $GIT_BINARY fetch --no-tags origin main dev
-            MAIN_TARGET="$($GIT_BINARY rev-parse origin/main)"
-            DEV_TARGET="$($GIT_BINARY rev-parse origin/dev)"
-            # Recovery can add CI/docs fixes after the immutable release tag.
-            # Both branches and every shipped application file must agree.
-            if $GIT_BINARY merge-base --is-ancestor "$MAIN_TARGET" "$DEV_TARGET" &&
-               $GIT_BINARY diff --quiet "$MAIN_TARGET" "$DEV_TARGET" &&
-               $GIT_BINARY diff --quiet "$RELEASE_SHA" "$MAIN_TARGET" -- . ':!.github' ':!README.md'; then
-              SYNC_DEV=true
-            fi
-          fi
-          {
-            echo 'dockerhub_tags<<EOF'
-            if [[ "$RELEASE_BRANCH" == dev ]]; then
-              echo "${DOCKERHUB_IMAGE}:dev"
-            elif [[ "$RELEASE_BRANCH" == main ]]; then
-              echo "${DOCKERHUB_IMAGE}:latest"
-              [[ "$SYNC_DEV" == true ]] && echo "${DOCKERHUB_IMAGE}:dev"
-            fi
-            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
-            echo 'EOF'
-            echo 'ghcr_tags<<EOF'
-            if [[ "$RELEASE_BRANCH" == dev ]]; then
-              echo "${GHCR_IMAGE}:dev"
-            elif [[ "$RELEASE_BRANCH" == main ]]; then
-              echo "${GHCR_IMAGE}:latest"
-              [[ "$SYNC_DEV" == true ]] && echo "${GHCR_IMAGE}:dev"
-            fi
-            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
-            echo 'EOF'
-          } >> "$GITHUB_OUTPUT"
-
-      - name: Create Docker Hub manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          TAG_ARGS=()
-          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
-          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Create GHCR manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          TAG_ARGS=()
-          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
-          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Verify published tags contain the exact tested images
-        env:
-          DOCKERHUB_TAGS: ${{ steps.docker_meta.outputs.dockerhub_tags }}
-          GHCR_TAGS: ${{ steps.docker_meta.outputs.ghcr_tags }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          # Both architectures already passed CLI/version/capture acceptance by
-          # digest. Verify promotion preserves those exact images and attestations
-          # instead of downloading and running the same image again on one arch.
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
-          EXPECTED="$("$DOCKER_BINARY" buildx imagetools create --dry-run "${REFS[@]}" | "$JQ_BINARY" -ceS '.manifests | sort_by(.digest)')"
-          "$JQ_BINARY" -e 'map(select(.platform.os == "linux") | .platform.architecture) | sort == ["amd64", "arm64"]' <<< "$EXPECTED"
-          while IFS= read -r image; do
-            [[ -n "$image" ]] || continue
-            ACTUAL="$("$DOCKER_BINARY" buildx imagetools inspect --raw "$image" | "$JQ_BINARY" -ceS 'select(.schemaVersion == 2) | .manifests | sort_by(.digest)')"
-            [[ "$ACTUAL" == "$EXPECTED" ]] || { echo "Published tag differs from tested images: $image" >&2; exit 1; }
-            echo "Verified exact tested images: $image"
-          done < <(printf '%s\n' "$DOCKERHUB_TAGS" "$GHCR_TAGS")
-
-      - name: Record published image source
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          # workflow_run can report the default branch's head even when this
-          # run published a different branch/SHA. Preserve the verified target.
-          [[ "$RELEASE_BRANCH" == dev || "$RELEASE_BRANCH" == main ]]
-          [[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]
-          mkdir -p "$RUNNER_TEMP/published-image-source"
-          # shellcheck disable=SC2016
-          "$JQ_BINARY" -n --arg branch "$RELEASE_BRANCH" --arg sha "$RELEASE_SHA" \
-            '{branch: $branch, sha: $sha}' > "$RUNNER_TEMP/published-image-source/source.json"
-
-      - name: Upload published image source
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: published-image-source
-          path: ${{ runner.temp }}/published-image-source/source.json
-          if-no-files-found: error
-
-      - name: Update Docker Hub README
-        uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: archivebox/archivebox
-
-  cascade:
-    name: Cascade verified release
-    if: needs.python-release.outputs.release_ready == 'true'
-    needs:
-      - candidate
-      - python-release
-      - docker-release
-    permissions:
-      actions: read
-      contents: read
-    uses: ArchiveBox/monorepo/.github/workflows/cascade-release.yml@main
-    with:
-      upstream_package: archivebox
-      upstream_version: ${{ needs.python-release.outputs.version }}
-      upstream_artifact_name: python-distributions
-      upstream_run_id: ${{ needs.candidate.outputs.artifact_run_id }}
-    secrets:
-      release_token: ${{ secrets.RELEASE_GH_TOKEN }}
@@ -1,81 +0,0 @@
-name: Capture publicsite screenshots
-
-on:
-  schedule:
-    - cron: '37 5,17 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-jobs:
-  changes:
-    runs-on: ubuntu-24.04
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      key: ${{ steps.check.outputs.key }}
-    steps:
-      - id: check
-        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
-
-  deploy:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Setup Python and uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: false
-
-      - name: Install ArchiveBox and screenshot dependencies
-        run: uv sync --no-cache --frozen
-
-      - name: Build UI screenshot gallery
-        run: bin/collect_ui_screenshots.sh
-        env:
-          BASE_URL: http://archivebox.localhost:5797
-
-      - name: Verify gallery was generated from this exact revision
-        run: |
-          uv run --no-cache --no-sync python - <<'PY'
-          import json
-          import os
-          import re
-          import tomllib
-          from pathlib import Path
-
-          provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
-          version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
-          assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
-          assert provenance["version"] == version, provenance
-          assert provenance["capture_count"] >= 3 * 35, provenance
-          assert len(provenance["files"]) == provenance["capture_count"], provenance
-          assert all(not re.match(r"^\d+-", name) for name in provenance["files"]), provenance
-          PY
-
-      - name: Save validated screenshots
-        uses: actions/upload-artifact@v4
-        with:
-          name: site-screenshots
-          path: publicsite/screenshots
-          if-no-files-found: error
-          retention-days: 90
-
-  complete:
-    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
-    needs: [changes, deploy]
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - run: echo 'Scheduled work completed successfully.'
@@ -1,117 +0,0 @@
-name: Staging acceptance
-
-on:
-  workflow_run:
-    workflows: [Release]
-    types: [completed]
-
-permissions: {}
-
-concurrency:
-  group: staging-acceptance
-  cancel-in-progress: false
-
-jobs:
-  published:
-    if: github.event.workflow_run.conclusion == 'success'
-    runs-on: ubuntu-24.04
-    permissions:
-      actions: read
-    outputs:
-      ready: ${{ steps.source.outputs.ready }}
-      sha: ${{ steps.source.outputs.sha }}
-    steps:
-      - id: image
-        env:
-          GH_TOKEN: ${{ github.token }}
-          RELEASE_RUN_ID: ${{ github.event.workflow_run.id }}
-        run: |
-          set -Eeuo pipefail
-          published=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RELEASE_RUN_ID/jobs?per_page=100" --jq 'any(.jobs[]; .name == "Publish tested multiarch image" and .conclusion == "success")')
-          echo "published=$published" >> "$GITHUB_OUTPUT"
-      - name: Download published image source
-        if: steps.image.outputs.published == 'true'
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: published-image-source
-          path: ${{ runner.temp }}/published-image-source
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ github.event.workflow_run.id }}
-      - name: Select published dev image
-        id: source
-        if: steps.image.outputs.published == 'true'
-        run: |
-          set -Eeuo pipefail
-          # The triggering run may report the default branch as its head;
-          # this artifact names the image source verified by the publisher.
-          source="$RUNNER_TEMP/published-image-source/source.json"
-          jq -e 'type == "object" and (keys | sort) == ["branch", "sha"]' "$source" >/dev/null
-          branch=$(jq -er '.branch | select(type == "string")' "$source")
-          sha=$(jq -er '.sha | select(type == "string")' "$source")
-          [[ "$branch" == dev || "$branch" == main ]]
-          [[ "$sha" =~ ^[0-9a-f]{40}$ ]]
-          echo "sha=$sha" >> "$GITHUB_OUTPUT"
-          if [[ "$branch" == dev ]]; then
-            echo 'ready=true' >> "$GITHUB_OUTPUT"
-          else
-            echo 'ready=false' >> "$GITHUB_OUTPUT"
-          fi
-  acceptance:
-    # The privileged key is never exposed to PRs or arbitrary dispatch refs.
-    needs: published
-    if: needs.published.outputs.ready == 'true'
-    runs-on: [self-hosted, Linux, X64, ugnas-linux]
-    environment: staging
-    timeout-minutes: 110
-    permissions:
-      contents: read
-      deployments: write
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [cabbage]
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
-        with:
-          ref: ${{ needs.published.outputs.sha }}
-          fetch-depth: 0
-          persist-credentials: false
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e
-        with:
-          version: '0.11.3'
-      - name: Prepare browser and restricted staging identity
-        env:
-          STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
-          STAGING_KNOWN_HOSTS: ${{ secrets.STAGING_KNOWN_HOSTS }}
-          ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/staging-browser
-        run: |
-          set -Eeuo pipefail
-          umask 077
-          printf '%s\n' "$STAGING_SSH_KEY" > "$RUNNER_TEMP/staging-key"
-          printf '%s\n' "$STAGING_KNOWN_HOSTS" > "$RUNNER_TEMP/staging-known-hosts"
-          # Use the candidate's normal provider/cache path, not a second
-          # Playwright-managed browser that bypasses abxpkg validation.
-          ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
-          CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
-          uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
-          CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
-          test -x "$CHROME_BINARY"
-          echo "CHROME_BINARY=$CHROME_BINARY" >> "$GITHUB_ENV"
-      - name: Verify actual deployment, full captures and rendered snapshots
-        env:
-          GH_TOKEN: ${{ github.token }}
-          STAGING_TARGET: ${{ matrix.target }}
-          STAGING_SHA: ${{ needs.published.outputs.sha }}
-          STAGING_KEY_FILE: ${{ runner.temp }}/staging-key
-          STAGING_KNOWN_HOSTS_FILE: ${{ runner.temp }}/staging-known-hosts
-        run: uv run --no-project --with playwright==1.63.0 python bin/check-staging-deployment.py
-      - name: Remove staging identity
-        if: always()
-        run: rm -f "$RUNNER_TEMP/staging-key" "$RUNNER_TEMP/staging-known-hosts"
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
-        if: always()
-        with:
-          name: staging-${{ matrix.target }}-${{ needs.published.outputs.sha }}
-          path: staging-evidence/
-          if-no-files-found: error
@@ -1,255 +1,118 @@
 name: Parallel Tests

 on:
-  workflow_dispatch:
-    inputs:
-      test_file:
-        description: Run one discovered test file
-        required: true
-        default: archivebox/tests/test_cli_list.py
-        type: string
-  workflow_call:
-    inputs:
-      test_file:
-        description: Run one discovered test file; empty runs the complete matrix
-        required: false
-        default: ""
-        type: string
+  pull_request:
+    branches: [dev, main, master]
+  push:
+    branches: [dev]

 env:
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False
-  UV_NO_SOURCES: "1"
-  CI_PYTHON_VERSION: "3.13"

 jobs:
  discover-tests:
-    name: Discover every test
-    # Selection must run on GitHub so an offline NAS cannot block the fallback.
-    runs-on: ubuntu-24.04
+    name: Discover test files
+    runs-on: ubuntu-22.04
    outputs:
-      tests: ${{ steps.discover.outputs.tests }}
-      runner: ${{ steps.runner.outputs.use-runner || '["ubuntu-24.04"]' }}
+      test-files: ${{ steps.set-matrix.outputs.test-files }}

    steps:
-      - name: Select available CI capacity
-        id: runner
-        if: vars.UGNAS_CI_ENABLED == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
-        uses: ArchiveBox/monorepo/.github/actions/select-ci-runner@main
-        with:
-          github-token: ${{ secrets.RUNNER_STATUS_TOKEN }}
-
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1

-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: ${{ env.CI_PYTHON_VERSION }}
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Resolve discovery tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+      - name: Discover test files
+        id: set-matrix
        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          test -n "$ABXPKG_SPEC"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PYTHON_BINARY="$pythonLocation/bin/python"
-          test -x "$PYTHON_BINARY"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
-          {
-            echo "PYTHON_BINARY=$ABXPKG_LIB_DIR/env/bin/python"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-          } >> "$GITHUB_ENV"
-          test -L "$ABXPKG_LIB_DIR/env/bin/python"
-          test -x "$ABXPKG_LIB_DIR/env/bin/python"
-          test -L "$ABXPKG_LIB_DIR/env/bin/jq"
-          test -x "$ABXPKG_LIB_DIR/env/bin/jq"
+          # Find all main test files
+          main_tests=$(find tests -maxdepth 1 -name "test_*.py" -type f | sort)

-      - name: Discover every test exactly once
-        id: discover
-        env:
-          TEST_FILE: ${{ inputs.test_file }}
-          UGNAS_CI_MAX_JOBS: ${{ vars.UGNAS_CI_MAX_JOBS || 3 }}
-        run: |
-          set -Eeuo pipefail
-          mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
-          test "${#output[@]}" -eq 2
-          if [[ -n "$TEST_FILE" ]]; then
-            output[1]="$("$JQ_BINARY" -ce '[.[] | select(.paths == [env.TEST_FILE])] | select(length == 1)' <<< "${output[1]}")"
-          fi
-          echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
-          echo "${output[0]}"
-          "$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"
+          # Find all plugin test files
+          plugin_tests=$(find archivebox/plugins -path "*/tests/test_*.py" -type f | sort)

-  tests:
+          # Combine and format as JSON array
+          all_tests=$(echo "$main_tests $plugin_tests" | tr ' ' '\n' | grep -v '^$')
+
+          # Create JSON array with test file info
+          json_array="["
+          first=true
+          for test_file in $all_tests; do
+            if [ "$first" = true ]; then
+              first=false
+            else
+              json_array+=","
+            fi
+
+            # Extract a display name for the test
+            if [[ $test_file == tests/* ]]; then
+              name="main/$(basename $test_file .py | sed 's/^test_//')"
+            else
+              plugin=$(echo $test_file | sed 's|archivebox/plugins/\([^/]*\)/.*|\1|')
+              test_name=$(basename $test_file .py | sed 's/^test_//')
+              name="plugin/$plugin/$test_name"
+            fi
+
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+          done
+          json_array+="]"
+
+          echo "test-files=$json_array" >> $GITHUB_OUTPUT
+          echo "Found $(echo $all_tests | wc -w) test files"
+          echo "$json_array" | jq '.'
+
+  run-tests:
    name: ${{ matrix.test.name }}
+    runs-on: ubuntu-22.04
    needs: discover-tests
-    # Opt in trusted branches only; PRs always use GitHub-hosted runners.
-    # Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
-    # The hosted discovery job falls back when workers are offline or the
-    # availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
-    runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
-    env:
-      CHROME_HEADLESS: "true"

    strategy:
      fail-fast: false
      matrix:
-        test: ${{ fromJson(needs.discover-tests.outputs.tests) }}
+        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
+        python: ["3.13"]

    steps:
-      - name: Set job-local runtime paths
-        env:
-          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
-        run: |
-          workers=2
-          cache_suffix=''
-          if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
-            workers="$(nproc)"
-            cache_suffix="-workers-$workers"
-            # Avoid the hosted image's cold Chromium executable page faults.
-            # abxpkg owns installation and reuse in the existing runtime cache.
-            echo "CHROME_BINPROVIDERS=playwright,puppeteer" >> "$GITHUB_ENV"
-          fi
-          {
-            echo "CI_TEST_WORKERS=$workers"
-            echo "ABXPKG_CACHE_WORKERS_SUFFIX=$cache_suffix"
-            echo "ABXPKG_LIB_DIR=$RUNNER_TEMP/archivebox-lib"
-            echo "DATA_DIR=$RUNNER_TEMP/archivebox-test-data"
-            echo "PERSONAS_DIR=$RUNNER_TEMP/abx-personas"
-            echo "CHROME_USER_DATA_DIR=$RUNNER_TEMP/abx-personas/Default/chrome_profile"
-          } >> "$GITHUB_ENV"
-
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1

-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v4
        with:
-          python-version: ${{ env.CI_PYTHON_VERSION }}
+          python-version: ${{ matrix.python }}
          architecture: x64
-        id: python

-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          version: "latest"

-      # Self-hosted tool storage survives jobs. Normal abxpkg validation and
-      # install locks allow reuse across files; cold-install tests use tmp_path.
-      - name: Reuse persistent runner tools
-        if: runner.environment == 'self-hosted'
-        env:
-          PERSISTENT_ABXPKG_LIB: ${{ runner.tool_cache }}/abxpkg/${{ github.repository }}/${{ steps.python.outputs.python-version }}
-        run: |
-          mkdir -p "$PERSISTENT_ABXPKG_LIB"
-          echo "ABXPKG_LIB_DIR=$PERSISTENT_ABXPKG_LIB" >> "$GITHUB_ENV"
-
-      # Restored binaries still pass normal abxpkg resolution/version checks.
-      # Installation and isolation tests explicitly choose fresh temporary libs.
-      - name: Identify runtime dependency cache
-        shell: bash
-        run: |
-          uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
-          import hashlib
-          import json
-          import tomllib
-          from pathlib import Path
-
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
-          print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
-          PYTHON
-
-      - name: Reuse validated runtime dependencies
-        if: runner.environment != 'self-hosted'
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
        with:
-          path: ${{ runner.temp }}/archivebox-lib
-          key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}${{ env.ABXPKG_CACHE_WORKERS_SUFFIX }}
+          node-version: 22
+
+      - name: Cache uv
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml', 'uv.lock') }}
          restore-keys: |
-            abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-
+            ${{ runner.os }}-${{ matrix.python }}-uv-

-      - name: Resolve optional build dependencies through abxpkg
-        if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
-        env:
-          TEST_EXTRA: ${{ matrix.test.extra }}
-          TEST_PATHS: ${{ matrix.test.paths_arg }}
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          test -n "$ABXPKG_SPEC"
-          dependency_sources=()
-          if [[ -n "$TEST_EXTRA" ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          fi
-          if [[ "$TEST_PATHS" == *test_postgres_backend* ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:postgres_binaries")
-          fi
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-            --install \
-            --no-cache \
-            --json \
-            --lib="$ABXPKG_LIB_DIR" \
-            "${dependency_sources[@]}"
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.1

-      - name: Install test dependencies
-        env:
-          TEST_EXTRA: ${{ matrix.test.extra }}
+      - name: Install dependencies with uv
        run: |
-          set -Eeuo pipefail
-          extra_args=()
-          if [[ -n "$TEST_EXTRA" ]]; then
-            extra_args=(--extra "$TEST_EXTRA")
-          fi
-          uv sync --no-cache --locked --dev "${extra_args[@]}"
-      - name: Run ${{ matrix.test.name }}
-        env:
-          TEST_PATHS: ${{ matrix.test.paths_arg }}
-          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
+          uv sync --dev --all-extras
+
+      - name: Run test - ${{ matrix.test.name }}
        run: |
-          set -Eeuo pipefail
-          echo "Running pytest with $CI_TEST_WORKERS workers on $CI_RUNNER_ENVIRONMENT"
-          read -r -a test_paths <<< "$TEST_PATHS"
-          test_args=()
-          for test_path in "${test_paths[@]}"; do
-            test -f "$test_path"
-            test_args+=("$GITHUB_WORKSPACE/$test_path")
-          done
-          # Each test still gets isolated DB/filesystem state from conftest.
-          # Keep one worker pool per batch instead of starting it for every file.
-          mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest"
-          uv run --no-cache --no-sync --no-sources pytest -vs -n "$CI_TEST_WORKERS" "${test_args[@]}" --durations=20 \
-            --basetemp="$RUNNER_TEMP/pytest" \
-            --junitxml="$RUNNER_TEMP/test-results/results.xml"
-      - name: Save test results
-        if: always()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: test-results-${{ strategy.job-index }}
-          path: ${{ runner.temp }}/test-results/*.xml
-          if-no-files-found: error
-          retention-days: 14
-      - name: Save failed runtime diagnostics
-        if: failure()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: runtime-logs-${{ strategy.job-index }}
-          path: ${{ runner.temp }}/pytest/**/*.log
-          if-no-files-found: ignore
-          retention-days: 14
+          uv run pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
@@ -1,60 +1,158 @@
-name: Install and CLI Compatibility
-
-on:
-  workflow_call:
-    inputs:
-      platform:
-        description: Platform for install and CLI compatibility
-        required: false
-        default: linux
-        type: string
+name: Run tests
+on: [push]

 env:
  DOCKER_IMAGE: archivebox-ci
-  ARCHIVEBOX_IMAGE: archivebox-ci
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False
-  UV_NO_SOURCES: "1"

 jobs:
-  install_cli_platform:
-    name: CLI install / ${{ matrix.os_name }} / Python ${{ matrix.python }}
+  python_tests:
    runs-on: ${{ matrix.os }}

    strategy:
-      fail-fast: false
      matrix:
-        include: ${{ fromJSON(inputs.platform == 'macos' && '[{"os":"macos-15","os_name":"macOS","python":"3.13"},{"os":"macos-15","os_name":"macOS","python":"3.14"}]' || '[{"os":"ubuntu-24.04","os_name":"linux","python":"3.14"}]') }}
+        os: [ubuntu-22.04]
+        # os: [ubuntu-22.04, macos-latest, windows-latest]
+        python: ["3.13"]

    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1

-      ### Set up Python
+      ### Setup Python & JS Languages
      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
+          architecture: x64

-      - name: Install uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          node-version: 20.10.0

-      - name: Install dependencies with uv
-        run: uv sync --no-cache --locked --dev
+      - name: Setup PDM
+        uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: '3.13'
+          cache: true

-      - name: ArchiveBox full install check
+      ### Install Python & JS Dependencies
+      - name: Get pip cache dir
+        id: pip-cache
        run: |
-          set -Eeuo pipefail
-          DATA_DIR="$PWD/tests/out/data"
-          mkdir -p "$DATA_DIR"
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox init
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox install
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox version
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox status
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: Cache pip
+        uses: actions/cache@v3
+        id: cache-pip
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python }}-venv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
+
+      - name: Install pip dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel pytest bottle build
+          python -m pip install -r requirements.txt
+          python -m pip install -e .[sonic,ldap]
+
+      - name: Get npm cache dir
+        id: npm-cache
+        run: |
+          echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
+
+      - name: Cache npm
+        uses: actions/cache@v3
+        id: cache-npm
+        with:
+          path: ${{ steps.npm-cache.outputs.dir }}
+          key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node_modules
+
+      - name: Install npm requirements
+        run: |
+          npm install
+          echo "SINGLEFILE_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/single-file" >> $GITHUB_ENV
+          echo "READABILITY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor" >> $GITHUB_ENV
+          echo "MERCURY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/mercury-parser" >> $GITHUB_ENV
+
+      ### Run the tests
+      - name: Directory listing for debugging
+        run: |
+          pwd
+          ls
+
+      - name: Archivebox version
+        run: |
+          archivebox version
+
+      - name: Test built package with pytest
+        # TODO: remove this exception for windows once we get tests passing on that platform
+        if: ${{ !contains(matrix.os, 'windows') }}
+        run: |
+          python -m pytest -s --basetemp=tests/out --ignore=archivebox/pkgs
+
+  docker_tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
+      # - uses: satackey/action-docker-layer-caching@v0.0.8
+
+      - name: Build image
+        run: |
+          docker build . -t "$DOCKER_IMAGE"
+
+      - name: Init data dir
+        run: |
+          mkdir "${{ github.workspace }}/data"
+          docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init
+
+      - name: Run test server
+        run: |
+          sudo bash -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
+          docker run --name www-nginx -p 80:80 -d nginx
+
+      - name: Add link
+        run: |
+          docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local
+
+      - name: Add stdin link
+        run: |
+          echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add
+
+      - name: List links
+        run: |
+          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
+          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }
+
+      - name: Start docker-compose stack
+        run: |
+          docker-compose run archivebox init
+          docker-compose up -d
+          sleep 5
+          curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
+          curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'
+
+      - name: Check added urls show up in index
+        run: |
+          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
+          curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
+          docker-compose down || true
@@ -1,22 +1,10 @@
 .DS_Store
 ._*
-/.heartbeat.json
-/package.json
-/package-lock.json
 *.pyc
 __pycache__/
 .mypy_cache/
 .eggs/
 tests/out/
-/docs/screenshots/
-/publicsite/screenshots/
-
-# Coverage
-.coverage
-.coverage.*
-coverage.json
-coverage/
-htmlcov/

 # Python and Node dependencies
 venv/
@@ -40,33 +28,16 @@ dist/

 # Data folders
 lib/
-out/
 tmp/
 data/
 data*/
-archivebox/tests/data/
-archive/
 output/
-logs/
-/-
-/personas/
-/sources/
 index.sqlite3
 queue.sqlite3
 *.sqlite*
 data.*
 .archivebox_id
-ArchiveBox.conf
-*.stdout
-*.stderr
-*.log
-.tmp/

 # vim
 *.sw?
 .vscode
-
-# Local Claude Code task locks (not project state)
-.claude/scheduled_tasks.lock
-
-/_site/
@@ -0,0 +1,3 @@
+[submodule "docs"]
+    path = docs
+    url = https://github.com/ArchiveBox/ArchiveBox.wiki.git
@@ -1,69 +0,0 @@
-default_language_version:
-  python: python3.13
-
-repos:
-  - repo: https://github.com/asottile/yesqa
-    rev: v1.5.0
-    hooks:
-      - id: yesqa
-
-  - repo: https://github.com/codespell-project/codespell
-    rev: v2.4.1
-    hooks:
-      - id: codespell
-        additional_dependencies:
-          - tomli
-
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.20.0
-    hooks:
-      - id: pyupgrade
-        args: [--py313-plus]
-
-  - repo: https://github.com/asottile/add-trailing-comma
-    rev: v3.1.0
-    hooks:
-      - id: add-trailing-comma
-
-  - repo: local
-    hooks:
-      - id: ruff-format
-        name: ruff-format
-        entry: uv run --active --no-sync --no-sources ruff format
-        language: system
-        types_or: [python, pyi]
-      - id: ruff-check
-        name: ruff-check
-        entry: uv run --active --no-sync --no-sources ruff check --fix
-        language: system
-        types_or: [python, pyi]
-
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
-    hooks:
-      - id: check-ast
-      - id: check-toml
-      - id: check-yaml
-        exclude: ^\.github/workflows/homebrew\.yml$
-      - id: check-json
-      - id: check-merge-conflict
-      - id: check-symlinks
-      - id: destroyed-symlinks
-      - id: check-case-conflict
-      - id: check-illegal-windows-names
-      - id: check-shebang-scripts-are-executable
-        exclude: ^(archivebox/.*\.py|archivebox/tests/.*\.py|archivebox/personas/export_browser_state\.js)$
-      - id: mixed-line-ending
-      - id: fix-byte-order-marker
-      - id: end-of-file-fixer
-        # Preserve the exact bytes of captured cryptographic proof fixtures.
-        exclude: ^archivebox/tests/fixtures/legacy_proofs/.*\.json$
-      - id: detect-private-key
-      - id: debug-statements
-      - id: forbid-submodules
-        exclude: ^docs$
-      - id: check-added-large-files
-        args: ["--maxkb=600"]
-      - id: name-tests-test
-        args: ["--pytest-test-first"]
-        exclude: ^archivebox/tests/(data/|fixtures\.py$|migrations_helpers\.py$)
@@ -1,83 +0,0 @@
-# ArchiveBox Agent Guide
-
-ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
-
-## Shared Standards
-
- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
- Start behavior fixes with a red failing test when a test is requested or practical.
- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
-
-## Concurrency Contract
-
- A collection has one orchestrator at a time. Local PID/process checks may warn about obvious same-machine duplicates, but must not claim to enforce ownership across machines or shared filesystems.
- SQLite remains supported with concurrent short writes from CLI and server processes. Keep the SQLite database on a local filesystem, never NFS/SMB, and use short autocommit/CAS updates instead of long transactions or database locks.
- Network calls, hook execution, filesystem migrations, and other long work belong in the orchestrator. Never hold a database transaction or lock across that work.
- PostgreSQL and shared data directories are the path to future multi-machine scheduling. Coordinate that work through database-backed per-crawl/per-snapshot claims; do not introduce file leases or timer-based orchestrator election.
-
-## Development Setup
-
-```bash
-uv sync --dev --all-extras
-mkdir -p data
-cd data
-uv run --project .. archivebox init --install
-```
-
-Run collection commands from inside an initialized data directory:
-
-```bash
-cd data
-uv run --project .. archivebox status
-uv run --project .. archivebox add --plugins=parse_txt_urls 'https://example.com/'
-uv run --project .. archivebox run
-```
-
-## User-Facing Setup
-
-Recommended CLI install:
-
-```bash
-uv tool install --python 3.13 --prerelease explicit --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
-mkdir -p ~/archivebox/data
-cd ~/archivebox/data
-archivebox init --install
-archivebox add --plugins=parse_txt_urls 'https://example.com/'
-```
-
-Alternative install methods:
-
- Docker Compose / Docker
- Homebrew
- Debian package
- pip
-
-## Basic Usage
-
-```bash
-cd ~/archivebox/data
-archivebox version
-archivebox help
-archivebox status
-archivebox install
-archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
-archivebox list --json --with-headers
-archivebox search 'example'
-archivebox update --filter-type=domain example.com
-archivebox remove --yes --delete --filter-type=exact 'https://example.com/docs-basic-usage'
-archivebox run
-```
-
-## Verification
-
-Run the test suite with `uv run pytest archivebox/tests -x` and repository checks
-with `uv run prek run --all-files`. For focused work, select the relevant tests.
-
-Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.
@@ -0,0 +1,3 @@
+[SERVER_CONFIG]
+SECRET_KEY = y6fw9wcaqls9sx_dze6ahky9ggpkpzoaw5g5v98_u3ro5j0_4f
+
@@ -0,0 +1,219 @@
+# Claude Code Development Guide for ArchiveBox
+
+## Quick Start
+
+```bash
+# Set up dev environment (always use uv, never pip directly)
+uv sync --dev --all-extras
+
+# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
+```
+
+## Development Environment Setup
+
+### Prerequisites
+- Python 3.11+ (3.13 recommended)
+- uv package manager
+- A non-root user for running tests (e.g., `testuser`)
+
+### Install Dependencies
+```bash
+uv sync --dev --all-extras  # Always use uv, never pip directly
+```
+
+### Activate Virtual Environment
+```bash
+source .venv/bin/activate
+```
+
+## Running Tests
+
+### CRITICAL: Never Run as Root
+ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
+
+```bash
+# Run all migration tests
+sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
+
+# Run specific test file
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
+
+# Run single test
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
+```
+
+### Test File Structure
+```
+archivebox/tests/
+├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
+├── test_migrations_fresh.py      # Fresh install tests
+├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
+├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
+└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
+```
+
+## Test Writing Standards
+
+### NO MOCKS - Real Tests Only
+Tests must exercise real code paths:
+- Create real SQLite databases with version-specific schemas
+- Seed with realistic test data
+- Run actual `python -m archivebox` commands via subprocess
+- Query SQLite directly to verify results
+
+**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
+
+### NO SKIPS
+Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
+
+### Strict Assertions
+- `init` command must return exit code 0 (not `[0, 1]`)
+- Verify ALL data is preserved, not just "at least one"
+- Use exact counts (`==`) not loose bounds (`>=`)
+
+### Example Test Pattern
+```python
+def test_migration_preserves_snapshots(self):
+    """Migration should preserve all snapshots."""
+    result = run_archivebox(self.work_dir, ['init'], timeout=45)
+    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
+
+    ok, msg = verify_snapshot_count(self.db_path, expected_count)
+    self.assertTrue(ok, msg)
+```
+
+## Migration Testing
+
+### Schema Versions
+- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
+- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
+- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
+- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
+
+### Testing a Migration Path
+1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
+2. Seed with realistic test data using `seed_0_X_data()`
+3. Run `archivebox init` to trigger migrations
+4. Verify data preservation with `verify_*` functions
+5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
+
+### Squashed Migrations
+When testing 0.8.x (dev branch), you must record ALL replaced migrations:
+```python
+# The squashed migration replaces these - all must be recorded
+('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
+('core', '0024_auto_20240513_1143'),
+# ... all 52 migrations from 0023-0074 ...
+('core', '0023_new_schema'),  # Also record the squashed migration itself
+```
+
+## Common Gotchas
+
+### 1. File Permissions
+New files created by root need permissions fixed for testuser:
+```bash
+chmod 644 archivebox/tests/test_*.py
+```
+
+### 2. DATA_DIR Environment Variable
+ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
+
+### 3. Extractors Disabled for Speed
+Tests disable all extractors via environment variables for faster execution:
+```python
+env['SAVE_TITLE'] = 'False'
+env['SAVE_FAVICON'] = 'False'
+# ... etc
+```
+
+### 4. Timeout Settings
+Use appropriate timeouts for migration tests (45s for init, 60s default).
+
+### 5. Circular FK References in Schemas
+SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
+
+## Architecture Notes
+
+### Crawl Model (0.9.x)
+- Crawl groups multiple Snapshots from a single `add` command
+- Each `add` creates one Crawl with one or more Snapshots
+- Seed model was removed - crawls now store URLs directly
+
+### Migration Strategy
+- Squashed migrations for clean installs
+- Individual migrations recorded for upgrades from dev branch
+- `replaces` attribute in squashed migrations lists what they replace
+
+## Code Style Guidelines
+
+### Naming Conventions for Grep-ability
+Use consistent naming for everything to enable easy grep-ability and logical grouping:
+
+**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
+
+**Examples**:
+```python
+# Filesystem migration methods - all start with fs_
+def fs_migration_needed() -> bool: ...
+def fs_migrate() -> None: ...
+def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
+def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
+def _fs_next_version(current: str) -> str: ...
+
+# Logging methods - ALL must start with log_ or _log
+def log_migration_start(snapshot_id: str) -> None: ...
+def _log_error(message: str) -> None: ...
+def log_validation_result(ok: bool, msg: str) -> None: ...
+```
+
+**Rules**:
+- Group related functions with common prefixes
+- Use `_` prefix for internal/private helpers within the same family
+- ALL logging-related methods MUST start with `log_` or `_log`
+- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
+- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
+
+### Minimize Unique Names and Data Structures
+**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
+
+**Example - GOOD**:
+```python
+# Binary has overrides field
+binary = Binary(overrides={'TIMEOUT': '60s'})
+
+# Binary reuses the same field name and structure
+class Binary(models.Model):
+    overrides = models.JSONField(default=dict)  # Same name, same structure
+```
+
+**Example - BAD**:
+```python
+# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
+class Binary(models.Model):
+    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
+```
+
+**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
+
+## Debugging Tips
+
+### Check Migration State
+```bash
+sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
+```
+
+### Check Table Schema
+```bash
+sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
+```
+
+### Verbose Test Output
+```bash
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
+```
+
+### Kill Zombie Chrome Processes
+```bash
+./bin/kill_chrome.sh
+```
@@ -1,230 +1,34 @@
-# syntax=docker/dockerfile:1.7
+# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
+#     python3.14, pip, pipx, uv, python3-ldap
+#     curl, wget, git, dig, ping, tree, nano
+#     node, npm, single-file, readability-extractor, postlight-parser
+#     ArchiveBox, yt-dlp, playwright, chromium
+# Usage:
+#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
+#     docker build . -t archivebox
+#     docker run -v "$PWD/data":/data archivebox init
+#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
+#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
+#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
+# Multi-arch build:
+#     docker buildx create --use
+#     docker buildx build . --platform=linux/amd64,linux/arm64--push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
+# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development

-# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# WHY: abx-dl owns Python, Node, Chromium, and extraction dependencies so the
-# standalone downloader stays independent of ArchiveBox's server features.
-# ArchiveBox-only dependencies (server plugins, supervisor, Django) belong in
-# the layers added by this Dockerfile, never in the shared abx-dl base image.
-# REQUIRED INSTALLATION CONTRACT — do not work around failures:
-# 1. IMAGE CONSTRUCTION: preinstall ALL plugins supported by runtime="archivebox"
-#    and ALL their required_binaries, regardless of default enabled state.
-#    Inherit the complete downloader dependencies from the tested abx-dl image;
-#    install every remaining ArchiveBox-only dependency in this image's layers,
-#    including optional server plugins. Never put server dependencies in abx-dl.
-#    Use the normal plugin dependency pipeline: install command -> config.json
-#    required_binaries -> abxpkg. Never add manual apt/pip/npm installs to hide
-#    failures in that pipeline. OS/interpreter bootstrap is a separate concern,
-#    not a second hand-maintained list of plugin dependencies.
-# 2. FINISHED IMAGE: archivebox install and the full plugin installation check
-#    must discover everything already present and be NO-OPS offline as UID 911.
-#    No downloads, package installs, sudo, or root repair may be needed to enable
-#    ANY plugin or start a new crawl. Never run verification as root or manually
-#    preinstall packages before it. If the inherited image lacks a downloader
-#    dependency, fix/release abx-dl and consume that tested image; do not patch
-#    over the missing base dependency here or repair deployed containers by SSH.
-# 3. NON-CONTAINER CI: exercise real runtime auto-installation through the same
-#    CLI/required_binaries/abxpkg path users invoke. GitHub's runner has
-#    passwordless sudo; abxpkg must escalate automatically when needed. Never
-#    manually preinstall dependencies or wrap the tested command in sudo to make
-#    a failure disappear. This is distinct from the finished-image no-op test.
-# 4. SIZE: the complete compressed image budget is 1025 MiB. Preserve ALL
-#    user-facing functionality. Only mechanical/internal
-#    optimizations are allowed. Never remove dependencies, disable features or
-#    plugins, weaken assertions, or skip checks to meet a size limit. If the
-#    complete image exceeds the configured limit, LET CI FAIL and explicitly
-#    ask the maintainer what to do. Do not raise the limit unilaterally.
-# Build abx-dl first, then point this file at it:
-#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
-#       --build-context abxbus=../abxbus \
-#       --build-context abxpkg=../abxpkg \
-#       --build-context abx-plugins=../abx-plugins \
-#       -t archivebox/abx-dl:dev
-#   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:dev \
-#       -t archivebox:multistage

-ARG ABX_DL_IMAGE=archivebox/abx-dl:main
+#########################################################################################

-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
+### Example: Using ArchiveBox in your own project's Dockerfile ########

-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-ARG ARCHIVEBOX_COMMIT_HASH=""
+# FROM python:3.14-slim
+# WORKDIR /data
+# RUN pip install archivebox>=0.8.5rc51   # use latest release here
+# RUN archivebox install
+# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data

-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_NO_COMPILE=1 \
-    PIP_ONLY_BINARY=aiohttp \
-    npm_config_loglevel=error
+#########################################################################################

-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=24
-
-ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_ARCHIVEBOX_UID=911 \
-    DEFAULT_ARCHIVEBOX_GID=911 \
-    IN_DOCKER=True
-
-ENV CODE_DIR=/app \
-    ABX_RUNTIME=archivebox \
-    DATA_DIR=/data \
-    CONFIG_DIR=/opt/archivebox \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
-    PERSONAS_DIR=/data/personas \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/venv/bin/python3 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-ENV HOME=/home/archivebox \
-    XDG_CONFIG_HOME=/home/archivebox/.config \
-    XDG_CACHE_HOME=/opt/archivebox/lib/cache \
-    TIMEOUT=600
-
-ENV UV_COMPILE_BYTECODE=false \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv \
-    VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:$PATH"
-
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
-WORKDIR "$CODE_DIR"
-
-RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
-    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env node \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env uv \
-    ) | tee -a /VERSION.txt
-
-FROM archivebox-runtime-base AS archivebox-builder
-
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
-    <<'EOF'
-echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
-export UV_NO_CACHE=true
-rm -f /etc/apt/apt.conf.d/docker-clean
-apt-get update -qq
-apt-get install -qq -y --no-install-recommends x11-utils
-# Extend abx-dl's existing venv instead of clearing it. Clearing and later
-# copying a complete replacement over the base duplicates every venv byte in
-# the final overlay history even when most packages are unchanged.
-# Do not rewrite inherited native libraries merely to strip them again.
-/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries
-
-# The old Docker-only dependency rewrite predated this checkout's lockfile:
-# it fetched version JSON to avoid stale PyPI Simple listings. That extra API
-# can itself lag or return 503s. The verified lock already records exact public
-# artifact URLs and hashes, including extras and platform markers. Consume it
-# directly, retaining the freshness check instead of resolving a second graph.
-/usr/bin/uv sync \
-    --project /app \
-    --locked \
-    --no-cache \
-    --no-dev \
-    --inexact \
-    --no-install-project \
-    --no-install-workspace \
-    --no-install-package abxbus \
-    --no-install-package abxpkg \
-    --no-install-package abx-plugins \
-    --no-install-package abx-dl \
-    --no-sources
-# Sync may replace inherited dependencies (e.g. pydantic/rich-click), removing
-# their bytecode. Runtime writes are disabled, so missing caches would make
-# every hook compile them again. Compile only missing entries: compileall's
-# timestamp fast path rewrites even valid checked-hash caches, copying the
-# entire inherited venv into this OCI layer for no runtime benefit.
-/usr/bin/uv run --no-project --no-sync python - <<'PY'
-import importlib.util
-from pathlib import Path
-import py_compile
-import sysconfig
-
-for source in Path(sysconfig.get_path("purelib")).rglob("*.py"):
-    cache = Path(importlib.util.cache_from_source(str(source)))
-    if not cache.is_file():
-        py_compile.compile(str(source), doraise=True, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH)
-PY
-builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
-ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
-/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
-while IFS= read -r -d '' native_library; do
-    if grep -Fxq "$native_library" /tmp/archivebox-inherited-libraries; then
-        continue
-    fi
-    magic=''
-    if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
-        "$builder_abxpkg_lib_dir/env/bin/strip" --strip-unneeded "$native_library" || exit $?
-    fi
-done < /tmp/archivebox-native-libraries
-rm -f /tmp/archivebox-native-libraries /tmp/archivebox-inherited-libraries
-rm -f /venv/bin/uv /venv/bin/uvx
-abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils
-abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get autoremove -y
-rm -rf "$builder_abxpkg_lib_dir"
-rm -rf /venv/lib/python3.*/site-packages/pip* \
-    /venv/lib/python3.*/site-packages/wheel* \
-    /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
-rm -rf /var/lib/apt/lists/*
-EOF
-
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-# Compile only ArchiveBox's installed package. Touching all of /venv here would
-# copy every inherited abx-dl file into a new layer merely to change its mtime.
-RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && COMMIT_HASH="$( \
-        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-            echo "$ARCHIVEBOX_COMMIT_HASH"; \
-        elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
-            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
-            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-                echo "$HEAD_REF"; \
-            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
-                REF_PATH="${HEAD_REF#ref: }"; \
-                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
-            fi; \
-        fi)" \
-    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
-    && /usr/bin/uv pip install --no-cache --no-deps "$CODE_DIR" \
-    && rm -f /venv/bin/uv /venv/bin/uvx \
-    && cd / \
-    && ARCHIVEBOX_PY_DIR="$(/venv/bin/python -c 'import pathlib, archivebox; print(pathlib.Path(archivebox.__file__).parent)')" \
-    && /venv/bin/python -m compileall --invalidation-mode checked-hash -q "$ARCHIVEBOX_PY_DIR" \
-    && test -f "$(/venv/bin/python -c 'import archivebox; print(archivebox.__cached__)')" \
-    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
-
-# The builder installs and purges stripping tools in one layer, so its final
-# filesystem is already runtime-clean. Preserve that ancestry: starting again
-# from archivebox-runtime-base and COPYing /venv would bake a second full venv
-# over the inherited one instead of recording only ArchiveBox's package delta.
-FROM archivebox-builder
+FROM ubuntu:24.04

 LABEL name="archivebox" \
    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -235,93 +39,361 @@ LABEL name="archivebox" \
    org.opencontainers.image.vendor="ArchiveBox" \
    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile"
+    com.docker.image.source.entrypoint="Dockerfile" \
+    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
+    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
+    com.docker.desktop.extension.api.version=">= 1.4.7" \
+    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
+    com.docker.extension.publisher-url="https://archivebox.io" \
+    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
+    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
+    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
+    com.docker.extension.categories='database,utility-tools'

-COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
+ARG TARGETPLATFORM
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
+######### Environment Variables #################################
+
+# Global built-time and runtime environment constants + default pkg manager config
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    npm_config_loglevel=error
+
+# Language Version config
+ENV PYTHON_VERSION=3.12 \
+    NODE_VERSION=22
+
+# Non-root User config
+ENV ARCHIVEBOX_USER="archivebox" \
+    DEFAULT_PUID=911 \
+    DEFAULT_PGID=911 \
+    IN_DOCKER=True
+
+# ArchiveBox Source Code + Lib + Data paths
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    PLAYWRIGHT_BROWSERS_PATH=/browsers
+    # GLOBAL_VENV=/venv \
+    # TODO: add TMP_DIR and LIB_DIR?
+
+# Bash SHELL config
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
+
+######### System Environment ####################################
+
+# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
+WORKDIR "$CODE_DIR"
+
+# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
+RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
+    && rm -f /etc/apt/apt.conf.d/docker-clean
+
+# Print debug info about build and save it to disk, for human eyes only, not used by anything else
+RUN (echo "[i] Docker build for ArchiveBox starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && echo \
+    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
+    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
+    && echo \
+    && uname -a \
+    && cat /etc/os-release | head -n7 \
+    && which bash && bash --version | head -n1 \
+    && which dpkg && dpkg --version | head -n1 \
+    && echo -e '\n\n' && env && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Create non-privileged user for archivebox and chrome
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
+    && groupadd --system $ARCHIVEBOX_USER \
+    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
+    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
+    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
+    | tee -a /VERSION.txt
+    # DEFAULT_PUID and DEFAULT_PID are overriden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
+    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
+
+# Install system apt dependencies (adding backports to access more recent apt updates)
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
+    && mkdir -p /etc/apt/keyrings \
+    && apt-get update -qq \
+    && apt-get install -qq -y \
+        # 1. packaging dependencies
+        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
+        # 2. docker and init system dependencies
+        zlib1g-dev dumb-init gosu cron unzip grep dnsutils \
+        # 3. frivolous CLI helpers to make debugging failed archiving easier
+        tree nano iputils-ping \
+        # nano iputils-ping dnsutils htop procps jq yq
+    && rm -rf /var/lib/apt/lists/*
+
+# Install apt binary dependencies for exractors
+# COPY --from=selenium/ffmpeg:latest /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing extractor dependencies for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        git ripgrep \
+        # Packages we have also needed in the past:
+        # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
+        # curl wget (already installed above)
+    && rm -rf /var/lib/apt/lists/* \
+    # Save version info
+    && ( \
+        which curl && curl --version | head -n1 \
+        && which wget && wget --version 2>&1 | head -n1 \
+        && which git && git --version 2>&1 | head -n1 \
+        # && which ffmpeg && (ffmpeg --version 2>&1 | head -n1) || true \
+        && which rg && rg --version 2>&1 | head -n1 \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install sonic search backend
+COPY --from=archivebox/sonic:1.4.9 /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
+RUN (which sonic && sonic --version) | tee -a /VERSION.txt

-# The builder invokes abxpkg as root for temporary ELF stripping, which can
-# rewrite derived state ownership. Restore UID 911 before resolving binaries
-# as the runtime user; the old copy-based final stage hid this dependency.
-RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777,id=archivebox-runtime-config-$TARGETARCH \
-    echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
-    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
-    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
-    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
-    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
-    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
-    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
-    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
-    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
-    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
-    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
-    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
-    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
-    && find "$ABXPKG_LIB_DIR" \( ! -user "$DEFAULT_ARCHIVEBOX_UID" -o ! -group "$DEFAULT_ARCHIVEBOX_GID" \) -exec chown -h "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" {} + \
-    && UV_NO_CACHE=true setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
-    && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+######### Language Environments ####################################

-# Install the complete ArchiveBox catalog, including disabled server plugins.
-# Explicit plugin names make installation independent of enabled defaults.
-RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-server-plugins-$TARGETARCH \
-    chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" /var/tmp/abxpkg-cache \
-    && chmod 1777 /var/tmp/abxpkg-cache \
-    && export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
-    && env ABXPKG_TMP_CACHE_DIR=/var/tmp/abxpkg-cache XDG_CACHE_HOME=/var/tmp/abxpkg-cache \
-        setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
-    && UV_NO_CACHE=true /usr/bin/uv run --no-project python -c 'from abx_plugins.plugins.opencode.image import prune_incompatible_image_files; prune_incompatible_image_files()' \
-    # pnpm's explicit store lives under ABXPKG_LIB_DIR, regardless of XDG_CACHE_HOME.
-    # Retaining its download copies made image size depend on whether the build
-    # preserved hardlinks to installed files. Remove the store in this layer;
-    # installed files retain their contents, and the offline check below proves
-    # the complete catalog still works without the disposable package payloads.
-    && rm -rf "$ABXPKG_LIB_DIR/cache/pnpm"
-RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
-    bash -c 'uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_installed; verify_installed()" && abx-dl install $ABX_DOCKER_PLUGINS'
+# Set up Python environment
+# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
+#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
+    # && apt-get update -qq \
+    # && apt-get install -qq -y --no-upgrade \
+    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
+    # && rm -rf /var/lib/apt/lists/* \
+    # tell PDM to allow using global system python site packages
+    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
+    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
+    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
+    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
+    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
+    # && python3 -m venv $GLOBAL_VENV \
+    # install global dependencies / python build dependencies in GLOBAL_VENV
+    # && pip install --upgrade pip setuptools wheel \
+    # Save version info
+    # && ( \
+    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
+    #     && which pip && pip --version \
+    #     # && which pdm && pdm --version \
+    #     && echo -e '\n\n' \
+    # ) | tee -a /VERSION.txt
+
+
+# Set up Node environment
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing NODE $NODE_VERSION for $TARGETPLATFORM..." \
+    && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
+    && curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-upgrade libatomic1 \
+    && apt-get install -y --no-upgrade \
+        nodejs \
+    && rm -rf /var/lib/apt/lists/* \
+    # Update NPM to latest version
+    && npm i -g npm --cache /root/.npm \
+    # Save version info
+    && ( \
+        which node && node --version \
+        && which npm && npm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+
+# Set up uv and main app /venv
+COPY --from=ghcr.io/astral-sh/uv:0.5 /uv /uvx /bin/
+ENV UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_PREFERENCE=only-system \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv
+WORKDIR "$CODE_DIR"
+# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM} (provided by base image)..." \
+    && uv python find --system \
+    && uv venv /venv
+ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
+RUN uv pip install setuptools pip \
+    && ( \
+        which python3 && python3 --version \
+        && which uv && uv version \
+        && uv python find --system && uv python find \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+
+######### ArchiveBox & Extractor Dependencies ##################################
+
+# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential gcc \
+        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
+        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
+    && uv pip install \
+        "python-ldap>=3.4.3" \
+    && apt-get purge -y \
+        python3-dev build-essential gcc \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+
+# Install apt font & rendering dependencies for chromium browser
+# TODO: figure out how much of this overlaps with `playwright install-deps chromium`
+# RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+
+# Install chromium browser binary using playwright
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing CHROMIUM dependencies, fonts, and display libraries for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y \
+        #fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
+        #at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
+        #libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
+        #libxaw7 libxcomposite1 libxdamage1 libxfont2 \
+        libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
+        # xfonts-scalable xfonts-utils xserver-common xvfb \
+        # chrome can run without dbus/upower technically, it complains about missing dbus but should run ok anyway
+        # libxss1 dbus dbus-x11 upower \
+    # && service dbus start \
+    && echo "[+] PIP Installing playwright into /venv and CHROMIUM binary into $PLAYWRIGHT_BROWSERS_PATH..." \
+    && uv pip install "playwright>=1.49.1" \
+    && uv run playwright install chromium --no-shell --with-deps \  
+    && export CHROME_BINARY="$(uv run python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
+    && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
+    && ln -s /browsers/ffmpeg-*/ffmpeg-linux /usr/bin/ffmpeg \
+    && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "/home/${ARCHIVEBOX_USER}/.config" \
+    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
+    && chown -R $ARCHIVEBOX_USER "$PLAYWRIGHT_BROWSERS_PATH" \
+    # delete extra full copy of node that playwright installs (saves >100mb)
+    && rm -f /venv/lib/python$PYTHON_VERSION/site-packages/playwright/driver/node \
+    # Save version info
+    && rm -rf /var/lib/apt/lists/* \
+    && ( \
+        uv pip show playwright \
+        && which chromium-browser && /usr/bin/chromium-browser --version || /usr/lib/chromium/chromium --version \
+        && which ffmpeg && ffmpeg -version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Install Node extractor dependencies
+ENV PATH="/home/$ARCHIVEBOX_USER/.npm/bin:$PATH"
+USER $ARCHIVEBOX_USER
+WORKDIR "/home/$ARCHIVEBOX_USER/.npm"
+RUN --mount=type=cache,target=/home/archivebox/.npm_cache,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT,uid=$DEFAULT_PUID,gid=$DEFAULT_PGID \
+    echo "[+] NPM Installing node extractor dependencies into /home/$ARCHIVEBOX_USER/.npm..." \
+    && npm config set prefix "/home/$ARCHIVEBOX_USER/.npm" \
+    && npm install --global --prefer-offline --no-fund --no-audit --cache "/home/$ARCHIVEBOX_USER/.npm_cache" \
+        "@postlight/parser@^2.2.3" \
+        "readability-extractor@github:ArchiveBox/readability-extractor" \
+        "single-file-cli@^1.1.54" \
+        "puppeteer@^23.5.0" \
+        "@puppeteer/browsers@^2.4.0" \
+    && rm -Rf "/home/$ARCHIVEBOX_USER/.cache/puppeteer"
+USER root
+WORKDIR "$CODE_DIR"
+RUN ( \
+        which node && node --version \
+        && which npm && npm version \
+        && which postlight-parser \
+        && which readability-extractor && readability-extractor --version \
+        && which single-file && single-file --version \
+        && which puppeteer && puppeteer --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+######### Build Dependencies ####################################
+
+
+# Install ArchiveBox Python venv dependencies from uv.lock
+RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml and uv.lock..." \
+    && uv sync \
+        --frozen \
+        --inexact \
+        --all-extras \
+        --no-install-project \
+        --no-install-workspace
+    # installs the pip packages that archivebox depends on, defined in pyproject.toml and uv.lock dependencies
+
+# Install ArchiveBox Python package + workspace dependencies from source
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && uv sync \
+        --frozen \
+        --inexact \
+        --all-extras \
+    && ( \
+        uv tree \
+        && which archivebox \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
+
+####################################################
+
+# Setup ArchiveBox runtime config
+ENV TMP_DIR=/tmp/archivebox \
+    LIB_DIR=/usr/share/archivebox/lib \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no

 WORKDIR "$DATA_DIR"
-RUN echo "[+] Initializing image collection..." \
-    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && HOME="$TMP_DIR" archivebox init \
-    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
-        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
-        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
-        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
-        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
-        "$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
-        2>/dev/null || true) \
-    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+RUN openssl rand -hex 16 > /etc/machine-id \
+    && mkdir -p "$TMP_DIR" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
+    && mkdir -p "$LIB_DIR" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
+    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt

-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
-
-RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
-    && test -f "$(/venv/bin/python -c 'import pydantic; print(pydantic.__cached__)')" \
-    && test -f "$(/venv/bin/python -c 'import rich_click; print(rich_click.__cached__)')" \
-    && test -f "$(/venv/bin/python -c 'import platformdirs; print(platformdirs.__cached__)')" \
-    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+# Print version for nice docker finish summary
+RUN (echo -e "\n\n[√] Finished Docker build succesfully. Saving build summary in: /VERSION.txt" \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
    ) | tee -a /VERSION.txt

+# Run   $ archivebox version                                >> /VERSION.txt
+# RUN "$CODE_DIR"/bin/docker_entrypoint.sh init 2>&1 | tee -a /VERSION.txt
+RUN "$CODE_DIR"/bin/docker_entrypoint.sh version 2>&1 | tee -a /VERSION.txt
+
+####################################################
+
+# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
-EXPOSE 5797
+EXPOSE 8000

 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:5797/health/' | grep -q 'OK'
+    CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'

 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:5797"]
+CMD ["archivebox", "server", "--quick-init", "0.0.0.0:8000"]
@@ -1,6 +1,6 @@
 MIT License

-Copyright (c) 2026 Nick Sweeting
+Copyright (c) 2024 Nick Sweeting

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -0,0 +1,532 @@
+# ArchiveBox Hook Script Concurrency & Execution Plan
+
+## Overview
+
+Snapshot.run() should enforce that snapshot hooks are run in **10 discrete, sequential "steps"**: `0*`, `1*`, `2*`, `3*`, `4*`, `5*`, `6*`, `7*`, `8*`, `9*`.
+
+For every discovered hook script, ArchiveBox should create an ArchiveResult in `queued` state, then manage running them using `retry_at` and inline logic to enforce this ordering.
+
+## Design Decisions
+
+### ArchiveResult Schema
+- Add `ArchiveResult.hook_name` (CharField, nullable) - just filename, e.g., `'on_Snapshot__20_chrome_tab.bg.js'`
+- Keep `ArchiveResult.plugin` - still important (plugin directory name)
+- Step number derived on-the-fly from `hook_name` via `extract_step(hook_name)` - not stored
+
+### Snapshot Schema
+- Add `Snapshot.current_step` (IntegerField 0-9, default=0)
+- Integrate with `SnapshotMachine` state transitions for step advancement
+
+### Hook Discovery & Execution
+- `Snapshot.run()` discovers all hooks upfront, creates one AR per hook with `hook_name` set
+- All ARs for a given step can be claimed and executed in parallel by workers
+- Workers claim ARs where `extract_step(ar.hook_name) <= snapshot.current_step`
+- `Snapshot.advance_step_if_ready()` increments `current_step` when:
+  - All **foreground** hooks in current step are finished (SUCCEEDED/FAILED/SKIPPED)
+  - Background hooks don't block advancement (they continue running)
+  - Called from `SnapshotMachine` state transitions
+
+### ArchiveResult.run() Behavior
+- If `self.hook_name` is set: run that single hook
+- If `self.hook_name` is None: discover all hooks for `self.plugin` and run sequentially
+- Background hooks detected by `.bg.` in filename (e.g., `on_Snapshot__20_chrome_tab.bg.js`)
+- Background hooks return immediately (ArchiveResult stays in STARTED state)
+- Foreground hooks wait for completion, update status from JSONL output
+
+### Hook Execution Flow
+1. **Within a step**: Workers claim all ARs for current step in parallel
+2. **Foreground hooks** (no .bg): ArchiveResult waits for completion, transitions to SUCCEEDED/FAILED/SKIPPED
+3. **Background hooks** (.bg): ArchiveResult transitions to STARTED, hook continues running
+4. **Step advancement**: `Snapshot.advance_step_if_ready()` checks:
+   - Are all foreground ARs in current step finished? (SUCCEEDED/FAILED/SKIPPED)
+   - Ignore ARs still in STARTED (background hooks)
+   - If yes, increment `current_step`
+5. **Snapshot sealing**: When `current_step=9` and all foreground hooks done, kill background hooks via `Snapshot.cleanup()`
+
+### Unnumbered Hooks
+- Extract step via `re.search(r'__(\d{2})_', hook_name)`, default to 9 if no match
+- Log warning for unnumbered hooks
+- Purely runtime derivation - no stored field
+
+## Hook Numbering Convention
+
+Hooks scripts are numbered `00` to `99` to control:
+- **First digit (0-9)**: Which step they are part of
+- **Second digit (0-9)**: Order within that step
+
+Hook scripts are launched **strictly sequentially** based on their filename alphabetical order, and run in sets of several per step before moving on to the next step.
+
+**Naming Format:**
+```
+on_{ModelName}__{run_order}_{human_readable_description}[.bg].{ext}
+```
+
+**Examples:**
+```
+on_Snapshot__00_this_would_run_first.sh
+on_Snapshot__05_start_ytdlp_download.bg.sh
+on_Snapshot__10_chrome_tab_opened.js
+on_Snapshot__50_screenshot.js
+on_Snapshot__53_media.bg.py
+```
+
+## Background (.bg) vs Foreground Scripts
+
+### Foreground Scripts (no .bg suffix)
+- Launch in parallel with other hooks in their step
+- Step waits for all foreground hooks to complete or timeout
+- Get killed with SIGTERM if they exceed their `PLUGINNAME_TIMEOUT`
+- Step advances when all foreground hooks finish
+
+### Background Scripts (.bg suffix)
+- Launch in parallel with other hooks in their step
+- Do NOT block step progression - step can advance while they run
+- Continue running across step boundaries until complete or timeout
+- Get killed with SIGTERM when Snapshot transitions to SEALED (via `Snapshot.cleanup()`)
+- Should exit naturally when work is complete (best case)
+
+**Important:** A .bg script started in step 2 can keep running through steps 3, 4, 5... until the Snapshot seals or the hook exits naturally.
+
+## Execution Step Guidelines
+
+These are **naming conventions and guidelines**, not enforced checkpoints. They provide semantic organization for plugin ordering:
+
+### Step 0: Pre-Setup
+```
+00-09: Initial setup, validation, feature detection
+```
+
+### Step 1: Chrome Launch & Tab Creation
+```
+10-19: Browser/tab lifecycle setup
+- Chrome browser launch
+- Tab creation and CDP connection
+```
+
+### Step 2: Navigation & Settlement
+```
+20-29: Page loading and settling
+- Navigate to URL
+- Wait for page load
+- Initial response capture (responses, ssl, consolelog as .bg listeners)
+```
+
+### Step 3: Page Adjustment
+```
+30-39: DOM manipulation before archiving
+- Hide popups/banners
+- Solve captchas
+- Expand comments/details sections
+- Inject custom CSS/JS
+- Accessibility modifications
+```
+
+### Step 4: Ready for Archiving
+```
+40-49: Final pre-archiving checks
+- Verify page is fully adjusted
+- Wait for any pending modifications
+```
+
+### Step 5: DOM Extraction (Sequential, Non-BG)
+```
+50-59: Extractors that need exclusive DOM access
+- singlefile (MUST NOT be .bg)
+- screenshot (MUST NOT be .bg)
+- pdf (MUST NOT be .bg)
+- dom (MUST NOT be .bg)
+- title
+- headers
+- readability
+- mercury
+
+These MUST run sequentially as they temporarily modify the DOM
+during extraction, then revert it. Running in parallel would corrupt results.
+```
+
+### Step 6: Post-DOM Extraction
+```
+60-69: Extractors that don't need DOM or run on downloaded files
+- wget
+- git
+- media (.bg - can run for hours)
+- gallerydl (.bg)
+- forumdl (.bg)
+- papersdl (.bg)
+```
+
+### Step 7: Chrome Cleanup
+```
+70-79: Browser/tab teardown
+- Close tabs
+- Cleanup Chrome resources
+```
+
+### Step 8: Post-Processing
+```
+80-89: Reprocess outputs from earlier extractors
+- OCR of images
+- Audio/video transcription
+- URL parsing from downloaded content (rss, html, json, txt, csv, md)
+- LLM analysis/summarization of outputs
+```
+
+### Step 9: Indexing & Finalization
+```
+90-99: Save to indexes and finalize
+- Index text content to Sonic/SQLite FTS
+- Create symlinks
+- Generate merkle trees
+- Final status updates
+```
+
+## Hook Script Interface
+
+### Input: CLI Arguments (NOT stdin)
+Hooks receive configuration as CLI flags (CSV or JSON-encoded):
+
+```bash
+--url="https://example.com"
+--snapshot-id="1234-5678-uuid"
+--config='{"some_key": "some_value"}'
+--plugins=git,media,favicon,title
+--timeout=50
+--enable-something
+```
+
+### Input: Environment Variables
+All configuration comes from env vars, defined in `plugin_dir/config.json` JSONSchema:
+
+```bash
+WGET_BINARY=/usr/bin/wget
+WGET_TIMEOUT=60
+WGET_USER_AGENT="Mozilla/5.0..."
+WGET_EXTRA_ARGS="--no-check-certificate"
+SAVE_WGET=True
+```
+
+**Required:** Every plugin must support `PLUGINNAME_TIMEOUT` for self-termination.
+
+### Output: Filesystem (CWD)
+Hooks read/write files to:
+- `$CWD`: Their own output subdirectory (e.g., `archive/snapshots/{id}/wget/`)
+- `$CWD/..`: Parent directory (to read outputs from other hooks)
+
+This allows hooks to:
+- Access files created by other hooks
+- Keep their outputs separate by default
+- Use semaphore files for coordination (if needed)
+
+### Output: JSONL to stdout
+Hooks emit one JSONL line per database record they want to create or update:
+
+```jsonl
+{"type": "Tag", "name": "sci-fi"}
+{"type": "ArchiveResult", "id": "1234-uuid", "status": "succeeded", "output_str": "wget/index.html"}
+{"type": "Snapshot", "id": "5678-uuid", "title": "Example Page"}
+```
+
+See `archivebox/misc/jsonl.py` and model `from_json()` / `from_jsonl()` methods for full list of supported types and fields.
+
+### Output: stderr for Human Logs
+Hooks should emit human-readable output or debug info to **stderr**. There are no guarantees this will be persisted long-term. Use stdout JSONL or filesystem for outputs that matter.
+
+### Cleanup: Delete Cruft
+If hooks emit no meaningful long-term outputs, they should delete any temporary files themselves to avoid wasting space. However, the ArchiveResult DB row should be kept so we know:
+- It doesn't need to be retried
+- It isn't missing
+- What happened (status, error message)
+
+### Signal Handling: SIGINT/SIGTERM
+Hooks are expected to listen for polite `SIGINT`/`SIGTERM` and finish hastily, then exit cleanly. Beyond that, they may be `SIGKILL'd` at ArchiveBox's discretion.
+
+**If hooks double-fork or spawn long-running processes:** They must output a `.pid` file in their directory so zombies can be swept safely.
+
+## Hook Failure Modes & Retry Logic
+
+Hooks can fail in several ways. ArchiveBox handles each differently:
+
+### 1. Soft Failure (Record & Don't Retry)
+**Exit:** `0` (success)
+**JSONL:** `{"type": "ArchiveResult", "status": "failed", "output_str": "404 Not Found"}`
+
+This means: "I ran successfully, but the resource wasn't available." Don't retry this.
+
+**Use cases:**
+- 404 errors
+- Content not available
+- Feature not applicable to this URL
+
+### 2. Hard Failure / Temporary Error (Retry Later)
+**Exit:** Non-zero (1, 2, etc.)
+**JSONL:** None (or incomplete)
+
+This means: "Something went wrong, I couldn't complete." Treat this ArchiveResult as "missing" and set `retry_at` for later.
+
+**Use cases:**
+- 500 server errors
+- Network timeouts
+- Binary not found / crashed
+- Transient errors
+
+**Behavior:**
+- ArchiveBox sets `retry_at` on the ArchiveResult
+- Hook will be retried during next `archivebox update`
+
+### 3. Partial Success (Update & Continue)
+**Exit:** Non-zero
+**JSONL:** Partial records emitted before crash
+
+**Behavior:**
+- Update ArchiveResult with whatever was emitted
+- Mark remaining work as "missing" with `retry_at`
+
+### 4. Success (Record & Continue)
+**Exit:** `0`
+**JSONL:** `{"type": "ArchiveResult", "status": "succeeded", "output_str": "output/file.html"}`
+
+This is the happy path.
+
+### Error Handling Rules
+
+- **DO NOT skip hooks** based on failures
+- **Continue to next hook** regardless of foreground or background failures
+- **Update ArchiveResults** with whatever information is available
+- **Set retry_at** for "missing" or temporarily-failed hooks
+- **Let background scripts continue** even if foreground scripts fail
+
+## File Structure
+
+```
+archivebox/plugins/{plugin_name}/
+├── config.json              # JSONSchema: env var config options
+├── binaries.jsonl           # Runtime dependencies: apt|brew|pip|npm|env
+├── on_Snapshot__XX_name.py  # Hook script (foreground)
+├── on_Snapshot__XX_name.bg.py  # Hook script (background)
+└── tests/
+    └── test_name.py
+```
+
+## Implementation Checklist
+
+### Phase 1: Schema Migration ✅
+- [x] Add `Snapshot.current_step` (IntegerField 0-9, default=0)
+- [x] Add `ArchiveResult.hook_name` (CharField, nullable) - just filename
+- [x] Create migration: `0034_snapshot_current_step.py`
+
+### Phase 2: Core Logic Updates ✅
+- [x] Add `extract_step(hook_name)` utility in `archivebox/hooks.py`
+  - Extract first digit from `__XX_` pattern
+  - Default to 9 for unnumbered hooks
+- [x] Add `is_background_hook(hook_name)` utility in `archivebox/hooks.py`
+  - Check for `.bg.` in filename
+- [x] Update `Snapshot.create_pending_archiveresults()` in `archivebox/core/models.py`:
+  - Discover all hooks (not plugins)
+  - Create one AR per hook with `hook_name` set
+- [x] Update `ArchiveResult.run()` in `archivebox/core/models.py`:
+  - If `hook_name` set: run single hook
+  - If `hook_name` None: discover all plugin hooks (existing behavior)
+- [x] Add `Snapshot.advance_step_if_ready()` method:
+  - Check if all foreground ARs in current step finished
+  - Increment `current_step` if ready
+  - Ignore background hooks (.bg) in completion check
+- [x] Integrate with `SnapshotMachine.is_finished()` in `archivebox/core/statemachines.py`:
+  - Call `advance_step_if_ready()` before checking if done
+
+### Phase 3: Worker Coordination ✅
+- [x] Update worker AR claiming query in `archivebox/workers/worker.py`:
+  - Filter: `extract_step(ar.hook_name) <= snapshot.current_step`
+  - Claims ARs in QUEUED state, checks step in Python before processing
+  - Orders by hook_name for deterministic execution within step
+
+### Phase 4: Hook Renumbering ✅
+- [x] Renumber hooks per renumbering map below
+- [x] Add `.bg` suffix to long-running hooks (media, gallerydl, forumdl, papersdl)
+- [x] Move parse_* hooks to step 7 (70-79)
+- [x] Test all hooks still work after renumbering
+
+## Migration Path
+
+### Natural Compatibility
+No special migration needed:
+1. Existing ARs with `hook_name=None` continue to work (discover all plugin hooks at runtime)
+2. New ARs get `hook_name` set (single hook per AR)
+3. `ArchiveResult.run()` handles both cases naturally
+4. Unnumbered hooks default to step 9 (log warning)
+
+### Renumbering Map
+
+**Completed Renames:**
+```
+# Step 5: DOM Extraction (sequential, non-background)
+singlefile/on_Snapshot__37_singlefile.py      → singlefile/on_Snapshot__50_singlefile.py ✅
+screenshot/on_Snapshot__34_screenshot.js      → screenshot/on_Snapshot__51_screenshot.js ✅
+pdf/on_Snapshot__35_pdf.js                    → pdf/on_Snapshot__52_pdf.js ✅
+dom/on_Snapshot__36_dom.js                    → dom/on_Snapshot__53_dom.js ✅
+title/on_Snapshot__32_title.js                → title/on_Snapshot__54_title.js ✅
+readability/on_Snapshot__52_readability.py    → readability/on_Snapshot__55_readability.py ✅
+headers/on_Snapshot__33_headers.js            → headers/on_Snapshot__55_headers.js ✅
+mercury/on_Snapshot__53_mercury.py            → mercury/on_Snapshot__56_mercury.py ✅
+htmltotext/on_Snapshot__54_htmltotext.py      → htmltotext/on_Snapshot__57_htmltotext.py ✅
+
+# Step 6: Post-DOM Extraction (background for long-running)
+wget/on_Snapshot__50_wget.py                  → wget/on_Snapshot__61_wget.py ✅
+git/on_Snapshot__12_git.py                    → git/on_Snapshot__62_git.py ✅
+media/on_Snapshot__51_media.py                → media/on_Snapshot__63_media.bg.py ✅
+gallerydl/on_Snapshot__52_gallerydl.py        → gallerydl/on_Snapshot__64_gallerydl.bg.py ✅
+forumdl/on_Snapshot__53_forumdl.py            → forumdl/on_Snapshot__65_forumdl.bg.py ✅
+papersdl/on_Snapshot__54_papersdl.py          → papersdl/on_Snapshot__66_papersdl.bg.py ✅
+
+# Step 7: URL Extraction (parse_* hooks moved from step 6)
+parse_html_urls/on_Snapshot__60_parse_html_urls.py      → parse_html_urls/on_Snapshot__70_parse_html_urls.py ✅
+parse_txt_urls/on_Snapshot__62_parse_txt_urls.py        → parse_txt_urls/on_Snapshot__71_parse_txt_urls.py ✅
+parse_rss_urls/on_Snapshot__61_parse_rss_urls.py        → parse_rss_urls/on_Snapshot__72_parse_rss_urls.py ✅
+parse_netscape_urls/on_Snapshot__63_parse_netscape_urls.py → parse_netscape_urls/on_Snapshot__73_parse_netscape_urls.py ✅
+parse_jsonl_urls/on_Snapshot__64_parse_jsonl_urls.py    → parse_jsonl_urls/on_Snapshot__74_parse_jsonl_urls.py ✅
+parse_dom_outlinks/on_Snapshot__40_parse_dom_outlinks.js → parse_dom_outlinks/on_Snapshot__75_parse_dom_outlinks.js ✅
+```
+
+## Testing Strategy
+
+### Unit Tests
+- Test hook ordering (00-99)
+- Test step grouping (first digit)
+- Test .bg vs foreground execution
+- Test timeout enforcement
+- Test JSONL parsing
+- Test failure modes & retry_at logic
+
+### Integration Tests
+- Test full Snapshot.run() with mixed hooks
+- Test .bg scripts running beyond step 99
+- Test zombie process cleanup
+- Test graceful SIGTERM handling
+- Test concurrent .bg script coordination
+
+### Performance Tests
+- Measure overhead of per-hook ArchiveResults
+- Test with 50+ concurrent .bg scripts
+- Test filesystem contention with many hooks
+
+## Open Questions
+
+### Q: Should we provide semaphore utilities?
+**A:** No. Keep plugins decoupled. Let them use simple filesystem coordination if needed.
+
+### Q: What happens if ArchiveResult table gets huge?
+**A:** We can delete old successful ArchiveResults periodically, or archive them to cold storage. The important data is in the filesystem outputs.
+
+### Q: Should naturally-exiting .bg scripts still be .bg?
+**A:** Yes. The .bg suffix means "don't block step progression," not "run until step 99." Natural exit is the best case.
+
+## Examples
+
+### Foreground Hook (Sequential DOM Access)
+```python
+#!/usr/bin/env python3
+# archivebox/plugins/screenshot/on_Snapshot__51_screenshot.js
+
+# Runs at step 5, blocks step progression until complete
+# Gets killed if it exceeds SCREENSHOT_TIMEOUT
+
+timeout = get_env_int('SCREENSHOT_TIMEOUT') or get_env_int('TIMEOUT', 60)
+
+try:
+    result = subprocess.run(cmd, capture_output=True, timeout=timeout)
+    if result.returncode == 0:
+        print(json.dumps({
+            "type": "ArchiveResult",
+            "status": "succeeded",
+            "output_str": "screenshot.png"
+        }))
+        sys.exit(0)
+    else:
+        # Temporary failure - will be retried
+        sys.exit(1)
+except subprocess.TimeoutExpired:
+    # Timeout - will be retried
+    sys.exit(1)
+```
+
+### Background Hook (Long-Running Download)
+```python
+#!/usr/bin/env python3
+# archivebox/plugins/ytdlp/on_Snapshot__63_ytdlp.bg.py
+
+# Runs at step 6, doesn't block step progression
+# Gets full YTDLP_TIMEOUT (e.g., 3600s) regardless of when step 99 completes
+
+timeout = get_env_int('YTDLP_TIMEOUT') or get_env_int('TIMEOUT', 3600)
+
+try:
+    result = subprocess.run(['yt-dlp', url], capture_output=True, timeout=timeout)
+    if result.returncode == 0:
+        print(json.dumps({
+            "type": "ArchiveResult",
+            "status": "succeeded",
+            "output_str": "media/"
+        }))
+        sys.exit(0)
+    else:
+        # Hard failure - don't retry
+        print(json.dumps({
+            "type": "ArchiveResult",
+            "status": "failed",
+            "output_str": "Video unavailable"
+        }))
+        sys.exit(0)  # Exit 0 to record the failure
+except subprocess.TimeoutExpired:
+    # Timeout - will be retried
+    sys.exit(1)
+```
+
+### Background Hook with Natural Exit
+```javascript
+#!/usr/bin/env node
+// archivebox/plugins/ssl/on_Snapshot__23_ssl.bg.js
+
+// Sets up listener, captures SSL info, then exits naturally
+// No SIGTERM handler needed - already exits when done
+
+async function main() {
+    const page = await connectToChrome();
+
+    // Set up listener
+    page.on('response', async (response) => {
+        const securityDetails = response.securityDetails();
+        if (securityDetails) {
+            fs.writeFileSync('ssl.json', JSON.stringify(securityDetails));
+        }
+    });
+
+    // Wait for navigation (done by other hook)
+    await waitForNavigation();
+
+    // Emit result
+    console.log(JSON.stringify({
+        type: 'ArchiveResult',
+        status: 'succeeded',
+        output_str: 'ssl.json'
+    }));
+
+    process.exit(0);  // Natural exit - no await indefinitely
+}
+
+main().catch(e => {
+    console.error(`ERROR: ${e.message}`);
+    process.exit(1);  // Will be retried
+});
+```
+
+## Summary
+
+This plan provides:
+- ✅ Clear execution ordering (10 steps, 00-99 numbering)
+- ✅ Async support (.bg suffix)
+- ✅ Independent timeout control per plugin
+- ✅ Flexible failure handling & retry logic
+- ✅ Streaming JSONL output for DB updates
+- ✅ Simple filesystem-based coordination
+- ✅ Backward compatibility during migration
+
+The main implementation work is refactoring `Snapshot.run()` to enforce step ordering and manage .bg script lifecycles. Plugin renumbering is straightforward mechanical work.
@@ -0,0 +1,172 @@
+# ArchiveBox UI
+
+## Page: Getting Started
+
+### What do you want to capture?
+
+- Save some URLs now -> [Add page]
+    - Paste some URLs to archive now
+    - Upload a file containing URLs (bookmarks.html export, RSS.xml feed, markdown file, word doc, PDF, etc.)
+    - Pull in URLs to archive from a remote location (e.g. RSS feed URL, remote TXT file, JSON file, etc.)
+
+- Import URLs from a browser -> [Import page]
+    - Desktop: Get the ArchiveBox Chrome/Firefox extension
+    - Mobile: Get the ArchiveBox iOS App / Android App
+    - Upload a bookmarks.html export file
+    - Upload a browser_history.sqlite3 export file
+
+- Import URLs from a 3rd party bookmarking service -> [Sync page]
+    - Pocket
+    - Pinboard
+    - Instapaper
+    - Wallabag
+    - Zapier, N8N, IFTTT, etc.
+    - Upload a bookmarks.html export, bookmarks.json, RSS, etc. file
+
+- Archive URLs on a schedule -> [Schedule page]
+
+- Archive an entire website -> [Crawl page]
+    - What starting URL/domain?
+    - How deep?
+    - Follow links to external domains?
+    - Follow links to parent URLs?
+    - Maximum number of pages to save?
+    - Maximum number of requests/minute?
+
+- Crawl for URLs with a search engine and save automatically
+    - 
+- Some URLs on a schedule
+- Save an entire website (e.g. `https://example.com`)
+- Save results matching a search query (e.g. "site:example.com")
+- Save a social media feed (e.g. `https://x.com/user/1234567890`)
+
+--------------------------------------------------------------------------------
+
+### Crawls App
+
+- Archive an entire website -> [Crawl page]
+    - What are the starting URLs?
+    - How many hops to follow?
+    - Follow links to external domains?
+    - Follow links to parent URLs?
+    - Maximum number of pages to save?
+    - Maximum number of requests/minute?
+
+
+--------------------------------------------------------------------------------
+
+### Scheduler App
+
+
+- Archive URLs on a schedule -> [Schedule page]
+    - What URL(s)?
+    - How often?
+    - Do you want to discard old snapshots after x amount of time?
+    - Any filter rules?
+    - Want to be notified when changes are detected -> redirect[Alerts app/create new alert(crawl=self)]
+
+
+* Choose Schedule check for new URLs: Schedule.objects.get(pk=xyz)
+    - 1 minute
+    - 5 minutes
+    - 1 hour
+    - 1 day
+
+    * Choose Destination Crawl to archive URLs using : Crawl.objects.get(pk=xyz)
+        - Tags
+        - Persona
+        - Created By ID
+        - Config
+        - Filters
+            - URL patterns to include
+            - URL patterns to exclude
+            - ONLY_NEW= Ignore URLs if already saved once / save URL each time it appears / only save is last save > x time ago
+
+
+--------------------------------------------------------------------------------
+
+### Sources App (For managing sources that ArchiveBox pulls URLs in from)
+
+- Add a new source to pull URLs in from (WIZARD)
+    - Choose URI:
+        - [x] Web UI
+        - [x] CLI
+        - Local filesystem path (directory to monitor for new files containing URLs)
+        - Remote URL (RSS/JSON/XML feed)
+        - Chrome browser profile sync (login using gmail to pull bookmarks/history)
+        - Pocket, Pinboard, Instapaper, Wallabag, etc.
+        - Zapier, N8N, IFTTT, etc.
+        - Local server filesystem path (directory to monitor for new files containing URLs)
+        - Google drive (directory to monitor for new files containing URLs)
+        - Remote server FTP/SFTP/SCP path (directory to monitor for new files containing URLs)
+        - AWS/S3/B2/GCP bucket (directory to monitor for new files containing URLs)
+        - XBrowserSync (login to pull bookmarks)
+    - Choose extractor
+        - auto
+        - RSS
+        - Pocket
+        - etc.
+    - Specify extra Config, e.g.
+        - credentials
+        - extractor tuning options (e.g. verify_ssl, cookies, etc.)
+
+- Provide credentials for the source
+    - API Key
+    - Username / Password
+    - OAuth
+
+--------------------------------------------------------------------------------
+
+### Alerts App
+
+- Create a new alert, choose condition
+    - Get notified when a site goes down (<x% success ratio for Snapshots)
+    - Get notified when a site changes visually more than x% (screenshot diff)
+    - Get notified when a site's text content changes more than x% (text diff)
+    - Get notified when a keyword appears
+    - Get notified when a keyword dissapears
+    - When an AI prompt returns some result
+
+- Choose alert threshold:
+    - any condition is met
+    - all conditions are met
+    - condition is met for x% of URLs
+    - condition is met for x% of time
+
+- Choose how to notify: (List[AlertDestination])
+    - maximum alert frequency
+    - destination type: email / Slack / Webhook / Google Sheet / logfile
+    - destination info:
+        - email address(es)
+        - Slack channel
+        - Webhook URL
+
+- Choose scope:
+    - Choose ArchiveResult scope (extractors): (a query that returns ArchiveResult.objects QuerySet)
+        - All extractors
+        - Only screenshots
+        - Only readability / mercury text
+        - Only video
+        - Only html
+        - Only headers
+
+    - Choose Snapshot scope (URL): (a query that returns Snapshot.objects QuerySet)
+        - All domains
+        - Specific domain
+        - All domains in a tag
+        - All domains in a tag category
+        - All URLs matching a certain regex pattern
+
+    - Choose crawl scope: (a query that returns Crawl.objects QuerySet)
+        - All crawls
+        - Specific crawls
+        - crawls by a certain user
+        - crawls using a certain persona
+
+
+class AlertDestination(models.Model):
+    destination_type: [email, slack, webhook, google_sheet, local logfile, b2/s3/gcp bucket, etc.]
+    maximum_frequency
+    filter_rules
+    credentials
+    alert_template: JINJA2 json/text template that gets populated with alert contents
@@ -9,23 +9,11 @@
 # in a universe that seems indifferent to us."
 # --Norber Weiner

-__package__ = "archivebox"
+__package__ = 'archivebox'

 import os
 import sys
 from pathlib import Path
-from typing import Protocol, cast
-
-
-class _ReconfigurableStream(Protocol):
-    def reconfigure(self, *, line_buffering: bool) -> object: ...
-
-
-# Force unbuffered output for real-time logs
-if hasattr(sys.stdout, "reconfigure"):
-    cast(_ReconfigurableStream, sys.stdout).reconfigure(line_buffering=True)
-    cast(_ReconfigurableStream, sys.stderr).reconfigure(line_buffering=True)
-os.environ["PYTHONUNBUFFERED"] = "1"

 ASCII_LOGO = """
 █████╗ ██████╗  ██████╗██╗  ██╗██╗██╗   ██╗███████╗ ██████╗  ██████╗ ██╗  ██╗
@@ -38,71 +26,49 @@ ASCII_LOGO = """

 PACKAGE_DIR = Path(__file__).resolve().parent

-os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
-os.environ["TZ"] = "UTC"
+# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models
+# # Migrations reference models like 'machine.Binary' which need to be importable
+# if str(PACKAGE_DIR) not in sys.path:
+#     sys.path.append(str(PACKAGE_DIR))
+
+os.environ['DJANGO_SETTINGS_MODULE'] = 'archivebox.core.settings'
+os.environ['TZ'] = 'UTC'

 # detect ArchiveBox user's UID/GID based on data dir ownership
-from .config.permissions import drop_privileges  # noqa
-
+from .config.permissions import drop_privileges                 # noqa
 drop_privileges()

-from .misc.checks import check_not_root, check_io_encoding  # noqa
-
+from .misc.checks import check_not_root, check_not_inside_source_dir, check_io_encoding      # noqa
 check_not_root()
+check_not_inside_source_dir()
 check_io_encoding()

-from .config.version import VERSION  # noqa
+# Install monkey patches for third-party libraries
+from .misc.monkey_patches import *                    # noqa

+# Built-in plugin directories
+BUILTIN_PLUGINS_DIR = PACKAGE_DIR / 'plugins'
+USER_PLUGINS_DIR = Path(os.getcwd()) / 'plugins'
+
+# These are kept for backwards compatibility with existing code
+# that checks for plugins. The new hook system uses discover_hooks()
+ALL_PLUGINS = {
+    'builtin': BUILTIN_PLUGINS_DIR,
+    'user': USER_PLUGINS_DIR,
+}
+LOADED_PLUGINS = ALL_PLUGINS
+
+# Setup basic config, constants, paths, and version
+from .config.constants import CONSTANTS                         # noqa
+from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR    # noqa
+from .config.version import VERSION                             # noqa
+
+# Set MACHINE_ID env var so hook scripts can use it
+os.environ.setdefault('MACHINE_ID', CONSTANTS.MACHINE_ID)

 __version__ = VERSION
-__author__ = "ArchiveBox"
-__license__ = "MIT"
-
-
-def __getattr__(name: str):
-    if name == "CONSTANTS":
-        from .config.constants import CONSTANTS
-
-        os.environ.setdefault("MACHINE_ID", CONSTANTS.MACHINE_ID)
-        return CONSTANTS
-    if name == "DATA_DIR":
-        from .config.paths import DATA_DIR
-
-        return DATA_DIR
-    if name == "VERSION":
-        return VERSION
-    if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
-        from abx_plugins import get_plugins_dir
-        from .config.constants import CONSTANTS
-
-        builtin_plugins_dir = Path(get_plugins_dir()).resolve()
-        user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
-        plugins = {
-            "builtin": builtin_plugins_dir,
-            "user": user_plugins_dir,
-        }
-        values = {
-            "BUILTIN_PLUGINS_DIR": builtin_plugins_dir,
-            "USER_PLUGINS_DIR": user_plugins_dir,
-            "ALL_PLUGINS": plugins,
-            "LOADED_PLUGINS": plugins,
-        }
-        return values[name]
-    raise AttributeError(name)
-
-
-__all__ = (
-    "ASCII_LOGO",
-    "ASCII_ICON",
-    "PACKAGE_DIR",
-    "DATA_DIR",
-    "CONSTANTS",
-    "VERSION",
-    "BUILTIN_PLUGINS_DIR",
-    "USER_PLUGINS_DIR",
-    "ALL_PLUGINS",
-    "LOADED_PLUGINS",
-)
+__author__ = 'ArchiveBox'
+__license__ = 'MIT'

 ASCII_ICON = """
 ██████████████████████████████████████████████████████████████████████████████████████████████████ 
@@ -1,9 +1,8 @@
 #!/usr/bin/env python3
 """This is the entrypoint for python -m archivebox ..."""
+__package__ = 'archivebox'

-__package__ = "archivebox"
-
-import archivebox  # noqa # make sure monkey patches are applied before anything else
+import archivebox      # noqa # make sure monkey patches are applied before anything else
 import sys

 from .cli import main
@@ -16,5 +15,5 @@ ASCII_LOGO_MINI = r"""
 /_/   \_\_|  \___|_| |_|_| \_/ \___|____/ \___/_/\_\
 """

-if __name__ == "__main__":
-    main(args=sys.argv[1:])
+if __name__ == '__main__':
+    main(args=sys.argv[1:], stdin=sys.stdin)
@@ -1 +1 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'
@@ -1,120 +1,68 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from django import forms
-from django.contrib import admin
-from django.http import HttpRequest
-from django.utils.text import capfirst
-from signal_webhooks.admin import WebhookAdmin, WebhookModelForm
-from signal_webhooks.settings import webhook_settings
-from signal_webhooks.utils import get_webhook_model, model_from_reference
+from signal_webhooks.admin import WebhookAdmin
+from signal_webhooks.utils import get_webhook_model

 from archivebox.base_models.admin import BaseModelAdmin

 from archivebox.api.models import APIToken


-def _webhook_fields(*names: str) -> tuple[str, ...]:
-    model_fields = {field.name for field in get_webhook_model()._meta.fields}
-    return tuple(name for name in names if name in model_fields)
-
-
 class APITokenAdmin(BaseModelAdmin):
-    list_display = ("created_at", "id", "created_by", "token_redacted", "expires")
-    sort_fields = ("id", "created_at", "created_by", "expires")
-    readonly_fields = ("created_at", "modified_at")
-    search_fields = ("id", "created_by__username", "token")
+    list_display = ('created_at', 'id', 'created_by', 'token_redacted', 'expires')
+    sort_fields = ('id', 'created_at', 'created_by', 'expires')
+    readonly_fields = ('created_at', 'modified_at')
+    search_fields = ('id', 'created_by__username', 'token')

    fieldsets = (
-        (
-            "Token",
-            {
-                "fields": ("token", "expires"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Owner",
-            {
-                "fields": ("created_by",),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": ("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
+        ('Token', {
+            'fields': ('token', 'expires'),
+            'classes': ('card',),
+        }),
+        ('Owner', {
+            'fields': ('created_by',),
+            'classes': ('card',),
+        }),
+        ('Timestamps', {
+            'fields': ('created_at', 'modified_at'),
+            'classes': ('card',),
+        }),
    )

-    list_filter = ("created_by",)
-    ordering = ["-created_at"]
+    list_filter = ('created_by',)
+    ordering = ['-created_at']
    list_per_page = 100


-class OutboundWebhookAdminForm(WebhookModelForm):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.fields["ref"] = forms.ChoiceField(
-            label=self.fields["ref"].label,
-            help_text=self.fields["ref"].help_text,
-            choices=[
-                (ref, f"{capfirst(model_from_reference(ref, check_hooks=False)._meta.verbose_name_plural)} ({ref})")
-                for ref in sorted(webhook_settings.HOOKS)
-            ],
-        )
-
-
 class CustomWebhookAdmin(WebhookAdmin, BaseModelAdmin):
-    form = OutboundWebhookAdminForm
-    list_display = ("created_at", "created_by", "id", *WebhookAdmin.list_display)
-    sort_fields = _webhook_fields("created_at", "created_by", "id", "ref", "endpoint", "last_success", "last_failure")
-    readonly_fields = _webhook_fields("created_at", "modified_at", *WebhookAdmin.readonly_fields)
+    list_display = ('created_at', 'created_by', 'id', *WebhookAdmin.list_display)
+    sort_fields = ('created_at', 'created_by', 'id', 'referenced_model', 'endpoint', 'last_success', 'last_error')
+    readonly_fields = ('created_at', 'modified_at', *WebhookAdmin.readonly_fields)

    fieldsets = (
-        (
-            "Webhook",
-            {
-                "fields": _webhook_fields("name", "signal", "ref", "endpoint", "headers", "keep_last_response"),
-                "classes": ("card", "wide"),
-            },
-        ),
-        (
-            "Authentication",
-            {
-                "fields": _webhook_fields("auth_token"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Status",
-            {
-                "fields": _webhook_fields("enabled", "last_success", "last_failure", "last_response"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Owner",
-            {
-                "fields": _webhook_fields("created_by"),
-                "classes": ("card",),
-            },
-        ),
-        (
-            "Timestamps",
-            {
-                "fields": _webhook_fields("created_at", "modified_at"),
-                "classes": ("card",),
-            },
-        ),
+        ('Webhook', {
+            'fields': ('name', 'signal', 'referenced_model', 'endpoint'),
+            'classes': ('card', 'wide'),
+        }),
+        ('Authentication', {
+            'fields': ('auth_token',),
+            'classes': ('card',),
+        }),
+        ('Status', {
+            'fields': ('enabled', 'last_success', 'last_error'),
+            'classes': ('card',),
+        }),
+        ('Owner', {
+            'fields': ('created_by',),
+            'classes': ('card',),
+        }),
+        ('Timestamps', {
+            'fields': ('created_at', 'modified_at'),
+            'classes': ('card',),
+        }),
    )

-    def lookup_allowed(self, lookup: str, value: str, request: HttpRequest | None = None) -> bool:
-        """Preserve WebhookAdmin's auth token filter with Django's current admin signature."""
-        return not lookup.startswith("auth_token") and admin.ModelAdmin.lookup_allowed(self, lookup, value, request)

-
-def register_admin(admin_site: admin.AdminSite) -> None:
+def register_admin(admin_site):
    admin_site.register(APIToken, APITokenAdmin)
    admin_site.register(get_webhook_model(), CustomWebhookAdmin)
@@ -1,19 +1,13 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

 from django.apps import AppConfig


 class APIConfig(AppConfig):
-    name = "archivebox.api"
-    label = "api"
-
-    def ready(self):
-        from archivebox.api.webhooks import register_delete_webhook_field_loader
-
-        register_delete_webhook_field_loader()
+    name = 'archivebox.api'
+    label = 'api'


 def register_admin(admin_site):
    from archivebox.api.admin import register_admin
-
    register_admin(admin_site)
@@ -1,19 +1,20 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

+from typing import Optional, cast
 from datetime import timedelta

-from django.utils import timezone
 from django.http import HttpRequest
+from django.utils import timezone
 from django.contrib.auth import authenticate
-from django.contrib.auth.models import User
+from django.contrib.auth.models import AbstractBaseUser

-from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader
+from ninja.security import HttpBearer, APIKeyQuery, APIKeyHeader, HttpBasicAuth
 from ninja.errors import HttpError


-def get_or_create_api_token(user: User | None):
+def get_or_create_api_token(user):
    from archivebox.api.models import APIToken
-
+    
    if user and user.is_superuser:
        api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
        if api_tokens.exists():
@@ -22,116 +23,116 @@ def get_or_create_api_token(user: User | None):
        else:
            # does not exist, create a new one
            api_token = APIToken.objects.create(created_by_id=user.pk, expires=timezone.now() + timedelta(days=30))
-
-        if api_token is None:
-            return None
+        
        assert api_token.is_valid(), f"API token is not valid {api_token}"

        return api_token
    return None


-def auth_using_token(token: str | None, request: HttpRequest | None = None) -> User | None:
+def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
    """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
-    from archivebox.api.models import APIToken  # lazy import model to avoid loading it at urls.py import time
+    from archivebox.api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
+    
+    user = None

-    user: User | None = None
-
-    submitted_empty_form = str(token).strip() in ("string", "", "None", "null")
+    submitted_empty_form = str(token).strip() in ('string', '', 'None', 'null')
    if not submitted_empty_form:
        try:
-            api_token = APIToken.objects.get(token=token)
-            if api_token.is_valid() and isinstance(api_token.created_by, User):
-                user = api_token.created_by
-                if request is not None:
-                    setattr(request, "_api_token", api_token)
+            token = APIToken.objects.get(token=token)
+            if token.is_valid():
+                user = token.created_by
+                request._api_token = token
        except APIToken.DoesNotExist:
            pass

-    return user
+    if not user:
+        # print('[❌] Failed to authenticate API user using API Key:', request)
+        return None
+    
+    return cast(AbstractBaseUser, user)

-
-def token_from_request(request: HttpRequest) -> str:
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-    return token
-
-
-def authenticated_user_from_request(request: HttpRequest) -> User | None:
-    user = request.user
-    if user.is_authenticated and user.is_active:
-        return user
-
-    token = token_from_request(request)
-    token_user = auth_using_token(token=token, request=request) if token else None
-    if token_user and token_user.is_active:
-        request.user = token_user
-        return token_user
-    return None
-
-
-def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
+def auth_using_password(username, password, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
    """Given a username and password, check if they are valid and return the corresponding user"""
-    user: User | None = None
-
-    submitted_empty_form = (username, password) in (("string", "string"), ("", ""), (None, None))
+    user = None
+    
+    submitted_empty_form = (username, password) in (('string', 'string'), ('', ''), (None, None))
    if not submitted_empty_form:
-        authenticated_user = authenticate(
+        user = authenticate(
            username=username,
            password=password,
        )
-        if isinstance(authenticated_user, User):
-            user = authenticated_user
-    return user
+
+    if not user:
+        # print('[❌] Failed to authenticate API user using API Key:', request)
+        user = None
+
+    return cast(AbstractBaseUser | None, user)


 ### Base Auth Types


-def _require_superuser(user: User | None, request: HttpRequest, auth_method: str) -> User | None:
-    if user and user.pk:
-        request.user = user
-        setattr(request, "_api_auth_method", auth_method)
-        if not user.is_superuser:
-            raise HttpError(403, "Valid credentials but User does not have permission (make sure user.is_superuser=True)")
-    return user
+class APITokenAuthCheck:
+    """The base class for authentication methods that use an api.models.APIToken"""
+    def authenticate(self, request: HttpRequest, key: Optional[str]=None) -> Optional[AbstractBaseUser]:
+        request.user = auth_using_token(
+            token=key,
+            request=request,
+        )
+        if request.user and request.user.pk:
+            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
+            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
+            request._api_auth_method = self.__class__.__name__
+
+            if not request.user.is_superuser:
+                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
+        return request.user
+
+
+class UserPassAuthCheck:
+    """The base class for authentication methods that use a username & password"""
+    def authenticate(self, request: HttpRequest, username: Optional[str]=None, password: Optional[str]=None) -> Optional[AbstractBaseUser]:
+        request.user = auth_using_password(
+            username=username,
+            password=password,
+            request=request,
+        )
+        if request.user and request.user.pk:
+            # Don't set cookie/persist login ouside this erquest, user may be accessing the API from another domain (CSRF/CORS):
+            # login(request, request.user, backend='django.contrib.auth.backends.ModelBackend')
+            request._api_auth_method = self.__class__.__name__
+
+            if not request.user.is_superuser:
+                raise HttpError(403, 'Valid API token but User does not have permission (make sure user.is_superuser=True)')
+
+        return request.user


 ### Django-Ninja-Provided Auth Methods

-
-class HeaderTokenAuth(APIKeyHeader):
+class HeaderTokenAuth(APITokenAuthCheck, APIKeyHeader):
    """Allow authenticating by passing X-API-Key=xyz as a request header"""
-
    param_name = "X-ArchiveBox-API-Key"

-    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
-        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
-
-
-class BearerTokenAuth(HttpBearer):
+class BearerTokenAuth(APITokenAuthCheck, HttpBearer):
    """Allow authenticating by passing Bearer=xyz as a request header"""
+    pass

-    def authenticate(self, request: HttpRequest, token: str) -> User | None:
-        return _require_superuser(auth_using_token(token=token, request=request), request, self.__class__.__name__)
-
-
-class QueryParamTokenAuth(APIKeyQuery):
+class QueryParamTokenAuth(APITokenAuthCheck, APIKeyQuery):
    """Allow authenticating by passing api_key=xyz as a GET/POST query parameter"""
-
    param_name = "api_key"

-    def authenticate(self, request: HttpRequest, key: str | None) -> User | None:
-        return _require_superuser(auth_using_token(token=key, request=request), request, self.__class__.__name__)
-
+class UsernameAndPasswordAuth(UserPassAuthCheck, HttpBasicAuth):
+    """Allow authenticating by passing username & password via HTTP Basic Authentication (not recommended)"""
+    pass

 ### Enabled Auth Methods

 API_AUTH_METHODS = [
    HeaderTokenAuth(),
    BearerTokenAuth(),
-    QueryParamTokenAuth(),
+    QueryParamTokenAuth(), 
    # django_auth_superuser,       # django admin cookie auth, not secure to use with csrf=False
+    UsernameAndPasswordAuth(),
 ]
@@ -1,32 +0,0 @@
-__package__ = "archivebox.api"
-
-from django.http import HttpResponse
-
-
-class ApiCorsMiddleware:
-    """Attach permissive CORS headers for API routes (token-based auth)."""
-
-    def __init__(self, get_response):
-        self.get_response = get_response
-
-    def __call__(self, request):
-        if request.path.startswith("/api/"):
-            if request.method == "OPTIONS" and request.META.get("HTTP_ACCESS_CONTROL_REQUEST_METHOD"):
-                response = HttpResponse(status=204)
-                return self._add_cors_headers(request, response)
-
-            response = self.get_response(request)
-            return self._add_cors_headers(request, response)
-
-        return self.get_response(request)
-
-    def _add_cors_headers(self, request, response):
-        origin = request.META.get("HTTP_ORIGIN")
-        if not origin:
-            return response
-
-        response["Access-Control-Allow-Origin"] = "*"
-        response["Access-Control-Allow-Methods"] = "GET, POST, PUT, PATCH, DELETE, OPTIONS"
-        response["Access-Control-Allow-Headers"] = "Authorization, X-ArchiveBox-API-Key, Content-Type, X-CSRFToken"
-        response["Access-Control-Max-Age"] = "600"
-        return response
@@ -12,9 +12,20 @@ import signal_webhooks.fields
 import signal_webhooks.utils


-# Raw sqlite DDL kept EXACTLY as-is; executed only on sqlite via Django's
-# RunSQL (identical statement splitting → byte-for-byte identical behavior).
-INITIAL_SQL = """
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('auth', '0012_alter_user_first_name_max_length'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    sql="""
                -- Create api_apitoken table
                CREATE TABLE IF NOT EXISTS api_apitoken (
                    id TEXT PRIMARY KEY NOT NULL,
@@ -60,222 +71,60 @@ INITIAL_SQL = """
                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
                CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
                CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
-                    """
-
-INITIAL_REVERSE_SQL = """
+                    """,
+                    reverse_sql="""
                DROP TABLE IF EXISTS api_outboundwebhook;
                DROP TABLE IF EXISTS api_apitoken;
                    """
-
-
-def _run_sqlite_only_sql(apps, schema_editor):
-    if schema_editor.connection.vendor != "sqlite":
-        return
-    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_forwards(
-        "api",
-        schema_editor,
-        None,
-        None,
-    )
-
-
-def _run_sqlite_only_sql_reverse(apps, schema_editor):
-    if schema_editor.connection.vendor != "sqlite":
-        return
-    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_backwards(
-        "api",
-        schema_editor,
-        None,
-        None,
-    )
-
-
-def _pg_sync_schema(apps, schema_editor):
-    from archivebox.misc.db import rebuild_models_from_migration_state
-
-    rebuild_models_from_migration_state(apps, schema_editor, "api", ["APIToken", "OutboundWebhook"])
-
-
-def _pg_drop_schema(apps, schema_editor):
-    from archivebox.misc.db import drop_models_on_postgres
-
-    drop_models_on_postgres(apps, schema_editor, "api", ["OutboundWebhook", "APIToken"])
-
-
-class Migration(migrations.Migration):
-    initial = True
-
-    dependencies = [
-        ("auth", "0012_alter_user_first_name_max_length"),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.SeparateDatabaseAndState(
-            database_operations=[
-                migrations.RunPython(_run_sqlite_only_sql, reverse_code=_run_sqlite_only_sql_reverse),
+                ),
            ],
            state_operations=[
                migrations.CreateModel(
-                    name="APIToken",
+                    name='APIToken',
                    fields=[
-                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
-                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
-                        ("modified_at", models.DateTimeField(auto_now=True)),
-                        ("token", models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
-                        ("expires", models.DateTimeField(blank=True, null=True)),
-                        (
-                            "created_by",
-                            models.ForeignKey(
-                                default=get_or_create_system_user_pk,
-                                on_delete=django.db.models.deletion.CASCADE,
-                                to=settings.AUTH_USER_MODEL,
-                            ),
-                        ),
+                        ('id', models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
+                        ('created_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+                        ('modified_at', models.DateTimeField(auto_now=True)),
+                        ('token', models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
+                        ('expires', models.DateTimeField(blank=True, null=True)),
+                        ('created_by', models.ForeignKey(default=get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
                    ],
                    options={
-                        "verbose_name": "API Key",
-                        "verbose_name_plural": "API Keys",
-                        "app_label": "api",
+                        'verbose_name': 'API Key',
+                        'verbose_name_plural': 'API Keys',
+                        'app_label': 'api',
                    },
                ),
                migrations.CreateModel(
-                    name="OutboundWebhook",
+                    name='OutboundWebhook',
                    fields=[
-                        ("id", models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
-                        ("created_at", models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
-                        ("modified_at", models.DateTimeField(auto_now=True)),
-                        (
-                            "name",
-                            models.CharField(db_index=True, help_text="Webhook name.", max_length=255, unique=True, verbose_name="name"),
-                        ),
-                        (
-                            "signal",
-                            models.CharField(
-                                choices=[
-                                    ("CREATE", "Create"),
-                                    ("UPDATE", "Update"),
-                                    ("DELETE", "Delete"),
-                                    ("M2M", "M2M changed"),
-                                    ("CREATE_OR_UPDATE", "Create or Update"),
-                                    ("CREATE_OR_DELETE", "Create or Delete"),
-                                    ("CREATE_OR_M2M", "Create or M2M changed"),
-                                    ("UPDATE_OR_DELETE", "Update or Delete"),
-                                    ("UPDATE_OR_M2M", "Update or M2M changed"),
-                                    ("DELETE_OR_M2M", "Delete or M2M changed"),
-                                    ("CREATE_UPDATE_OR_DELETE", "Create, Update or Delete"),
-                                    ("CREATE_UPDATE_OR_M2M", "Create, Update or M2M changed"),
-                                    ("CREATE_DELETE_OR_M2M", "Create, Delete or M2M changed"),
-                                    ("UPDATE_DELETE_OR_M2M", "Update, Delete or M2M changed"),
-                                    ("CREATE_UPDATE_DELETE_OR_M2M", "Create, Update or Delete, or M2M changed"),
-                                ],
-                                help_text="Signal the webhook fires to.",
-                                max_length=255,
-                                verbose_name="signal",
-                            ),
-                        ),
-                        (
-                            "ref",
-                            models.CharField(
-                                db_index=True,
-                                help_text="Dot import notation to the model the webhook is for.",
-                                max_length=1023,
-                                validators=[signal_webhooks.utils.model_from_reference],
-                                verbose_name="referenced model",
-                            ),
-                        ),
-                        (
-                            "endpoint",
-                            models.URLField(help_text="Target endpoint for this webhook.", max_length=2047, verbose_name="endpoint"),
-                        ),
-                        (
-                            "headers",
-                            models.JSONField(
-                                blank=True,
-                                default=dict,
-                                help_text="Headers to send with the webhook request.",
-                                validators=[signal_webhooks.utils.is_dict],
-                                verbose_name="headers",
-                            ),
-                        ),
-                        (
-                            "auth_token",
-                            signal_webhooks.fields.TokenField(
-                                blank=True,
-                                default="",
-                                help_text="Authentication token to use in an Authorization header.",
-                                max_length=8000,
-                                validators=[signal_webhooks.utils.decode_cipher_key],
-                                verbose_name="authentication token",
-                            ),
-                        ),
-                        ("enabled", models.BooleanField(default=True, help_text="Is this webhook enabled?", verbose_name="enabled")),
-                        (
-                            "keep_last_response",
-                            models.BooleanField(
-                                default=False,
-                                help_text="Should the webhook keep a log of the latest response it got?",
-                                verbose_name="keep last response",
-                            ),
-                        ),
-                        (
-                            "created",
-                            models.DateTimeField(auto_now_add=True, help_text="When the webhook was created.", verbose_name="created"),
-                        ),
-                        (
-                            "updated",
-                            models.DateTimeField(auto_now=True, help_text="When the webhook was last updated.", verbose_name="updated"),
-                        ),
-                        (
-                            "last_response",
-                            models.CharField(
-                                blank=True,
-                                default="",
-                                help_text="Latest response to this webhook.",
-                                max_length=8000,
-                                verbose_name="last response",
-                            ),
-                        ),
-                        (
-                            "last_success",
-                            models.DateTimeField(
-                                default=None,
-                                help_text="When the webhook last succeeded.",
-                                null=True,
-                                verbose_name="last success",
-                            ),
-                        ),
-                        (
-                            "last_failure",
-                            models.DateTimeField(
-                                default=None,
-                                help_text="When the webhook last failed.",
-                                null=True,
-                                verbose_name="last failure",
-                            ),
-                        ),
-                        (
-                            "created_by",
-                            models.ForeignKey(
-                                default=get_or_create_system_user_pk,
-                                on_delete=django.db.models.deletion.CASCADE,
-                                to=settings.AUTH_USER_MODEL,
-                            ),
-                        ),
+                        ('id', models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
+                        ('created_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
+                        ('modified_at', models.DateTimeField(auto_now=True)),
+                        ('name', models.CharField(db_index=True, help_text='Webhook name.', max_length=255, unique=True, verbose_name='name')),
+                        ('signal', models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='Signal the webhook fires to.', max_length=255, verbose_name='signal')),
+                        ('ref', models.CharField(db_index=True, help_text='Dot import notation to the model the webhook is for.', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model')),
+                        ('endpoint', models.URLField(help_text='Target endpoint for this webhook.', max_length=2047, verbose_name='endpoint')),
+                        ('headers', models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers')),
+                        ('auth_token', signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token')),
+                        ('enabled', models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled')),
+                        ('keep_last_response', models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response')),
+                        ('created', models.DateTimeField(auto_now_add=True, help_text='When the webhook was created.', verbose_name='created')),
+                        ('updated', models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated')),
+                        ('last_response', models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response')),
+                        ('last_success', models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success')),
+                        ('last_failure', models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure')),
+                        ('created_by', models.ForeignKey(default=get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
                    ],
                    options={
-                        "verbose_name": "API Outbound Webhook",
-                        "app_label": "api",
+                        'verbose_name': 'API Outbound Webhook',
+                        'app_label': 'api',
                    },
                ),
                migrations.AddConstraint(
-                    model_name="outboundwebhook",
-                    constraint=models.UniqueConstraint(fields=["ref", "endpoint"], name="prevent_duplicate_hooks_api_outboundwebhook"),
+                    model_name='outboundwebhook',
+                    constraint=models.UniqueConstraint(fields=['ref', 'endpoint'], name='prevent_duplicate_hooks_api_outboundwebhook'),
                ),
            ],
        ),
-        # On non-sqlite backends the raw DDL above is skipped, so the real
-        # schema diverges from Django state. Rebuild both tables (including the
-        # unique constraint) from the post-migration state (no-op on sqlite).
-        migrations.RunPython(_pg_sync_schema, reverse_code=_pg_drop_schema),
    ]
@@ -1,7 +1,8 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

 import secrets
-from archivebox.uuid_compat import CompactUUIDField, uuid7
+from archivebox.uuid_compat import uuid7
+from datetime import timedelta

 from django.conf import settings
 from django.db import models
@@ -17,7 +18,7 @@ def generate_secret_token() -> str:


 class APIToken(models.Model):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)
@@ -25,7 +26,7 @@ class APIToken(models.Model):
    expires = models.DateTimeField(null=True, blank=True)

    class Meta(TypedModelMeta):
-        app_label = "api"
+        app_label = 'api'
        verbose_name = "API Key"
        verbose_name_plural = "API Keys"

@@ -34,21 +35,21 @@ class APIToken(models.Model):

    @property
    def token_redacted(self):
-        return f"************{self.token[-4:]}"
+        return f'************{self.token[-4:]}'

    def is_valid(self, for_date=None):
        return not self.expires or self.expires >= (for_date or timezone.now())


 class OutboundWebhook(WebhookBase):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)

    class Meta(WebhookBase.Meta):
-        app_label = "api"
-        verbose_name = "API Outbound Webhook"
+        app_label = 'api'
+        verbose_name = 'API Outbound Webhook'

    def __str__(self) -> str:
-        return f"[{self.id}] {self.ref} -> {self.endpoint}"
+        return f'[{self.id}] {self.ref} -> {self.endpoint}'
@@ -0,0 +1,30 @@
+__package__ = 'archivebox.api'
+
+# from django.test import TestCase
+# from ninja.testing import TestClient
+
+# from .routes_cli import router
+
+# class ArchiveBoxCLIAPITestCase(TestCase):
+#     def setUp(self):
+#         self.client = TestClient(router)
+
+#     def test_add_endpoint(self):
+#         response = self.client.post("/add", json={"urls": ["http://example.com"], "tag": "testTag1,testTag2"})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
+
+#     def test_remove_endpoint(self):
+#         response = self.client.post("/remove", json={"filter_patterns": ["http://example.com"]})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
+
+#     def test_update_endpoint(self):
+#         response = self.client.post("/update", json={})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
+
+#     def test_list_all_endpoint(self):
+#         response = self.client.post("/list_all", json={})
+#         self.assertEqual(response.status_code, 200)
+#         self.assertTrue(response.json()["success"])
@@ -1,26 +1,17 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from urllib.parse import quote
-
-from django.http import HttpRequest, HttpResponseRedirect
-from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView

-from archivebox.core.routes_util import build_web_url
 from .v1_api import urls as v1_api_urls

-
-def archive_redirect_view(request: HttpRequest, url: str) -> HttpResponseRedirect:
-    if request.META.get("QUERY_STRING"):
-        url = f"{url}?{request.META['QUERY_STRING']}"
-    return redirect(build_web_url(f"/web/{quote(url, safe=':/')}", request=request))
-
-
 urlpatterns = [
-    path("", RedirectView.as_view(url="/api/v1/docs")),
-    path("archive/<path:url>", archive_redirect_view, name="api-archive-redirect"),
-    path("v1/", RedirectView.as_view(url="/api/v1/docs")),
-    path("v1/", v1_api_urls),
-    path("v1", RedirectView.as_view(url="/api/v1/docs")),
+    path("",                 RedirectView.as_view(url='/api/v1')),
+
+    path("v1/",              v1_api_urls),
+    path("v1",               RedirectView.as_view(url='/api/v1/docs')),
+
+    # ... v2 can be added here ...
+    # path("v2/",              v2_api_urls),
+    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
 ]
@@ -1,18 +1,14 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'


 from io import StringIO
-from functools import lru_cache
-from typing import Any
 from traceback import format_exception
 from contextlib import redirect_stdout, redirect_stderr

 from django.http import HttpRequest, HttpResponse
 from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, PermissionDenied
-from django.contrib.auth.models import User

 from ninja import NinjaAPI, Swagger
-from ninja.openapi.schema import OpenAPISchema

 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/

@@ -20,12 +16,11 @@ from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH

 from archivebox.api.auth import API_AUTH_METHODS
-from archivebox.api.models import APIToken


-COMMIT_HASH = get_COMMIT_HASH() or "unknown"
+COMMIT_HASH = get_COMMIT_HASH() or 'unknown'

-html_description = f"""
+html_description=f'''
 <h3>Welcome to your ArchiveBox server's REST API <code>[v1 ALPHA]</code> homepage!</h3>
 <br/>
 <i><b>WARNING: This API is still in an early development stage and may change!</b></i>
@@ -38,69 +33,63 @@ html_description = f"""
 <li>📜 See the API source code: <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/api"><code>archivebox/api/</code></a></li>
 </ul>
 <small>Served by ArchiveBox v{VERSION} (<a href="https://github.com/ArchiveBox/ArchiveBox/commit/{COMMIT_HASH}"><code>{COMMIT_HASH[:8]}</code></a>), API powered by <a href="https://django-ninja.dev/"><code>django-ninja</code></a>.</small>
-"""
+'''


 def register_urls(api: NinjaAPI) -> NinjaAPI:
-    api.add_router("/auth/", "archivebox.api.v1_auth.router")
-    api.add_router("/core/", "archivebox.api.v1_core.router")
-    api.add_router("/crawls/", "archivebox.api.v1_crawls.router")
-    api.add_router("/cli/", "archivebox.api.v1_cli.router")
-    api.add_router("/machine/", "archivebox.api.v1_machine.router")
-    api.add_router("/personas/", "archivebox.api.v1_personas.router")
+    # api.add_router('/auth/',     'archivebox.api.v1_auth.router')
+    api.add_router('/core/',     'archivebox.api.v1_core.router')
+    api.add_router('/crawls/',   'archivebox.api.v1_crawls.router')
+    api.add_router('/cli/',      'archivebox.api.v1_cli.router')
+    api.add_router('/workers/',  'archivebox.api.v1_workers.router')
+    api.add_router('/machine/',  'archivebox.api.v1_machine.router')
    return api


-class NinjaAPIWithIOCapture(NinjaAPI):
-    def get_openapi_schema(self, *, path_prefix: str | None = None, path_params: dict[str, Any] | None = None) -> OpenAPISchema:
-        prefix = path_prefix if path_prefix is not None else self.get_root_path(path_params or {})
-        return self._schema_for_prefix(prefix)
-
-    @lru_cache(maxsize=8)
-    def _schema_for_prefix(self, path_prefix: str) -> OpenAPISchema:
-        # Routes are registered at startup. Server discovery can reuse their
-        # schema while preserving distinct URL prefixes for mounted instances.
-        return super().get_openapi_schema(path_prefix=path_prefix)
-
+class NinjaAPIWithIOCapture(NinjaAPI):    
    def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
        stdout, stderr = StringIO(), StringIO()

        with redirect_stderr(stderr):
            with redirect_stdout(stdout):
-                setattr(request, "stdout", stdout)
-                setattr(request, "stderr", stderr)
+                request.stdout = stdout
+                request.stderr = stderr

                response = super().create_temporal_response(request)

-        # Disable caching of API responses entirely
-        response["Cache-Control"] = "no-store"
+        # Diable caching of API responses entirely
+        response['Cache-Control'] = 'no-store'

        # Add debug stdout and stderr headers to response
-        response["X-ArchiveBox-Stdout"] = stdout.getvalue().replace("\n", "\\n")[:200]
-        response["X-ArchiveBox-Stderr"] = stderr.getvalue().replace("\n", "\\n")[:200]
+        response['X-ArchiveBox-Stdout'] = str(request.stdout)[200:]
+        response['X-ArchiveBox-Stderr'] = str(request.stderr)[200:]
        # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'

        # Add Auth Headers to response
-        api_token_attr = request.__dict__.get("_api_token")
-        api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
-        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
+        api_token = getattr(request, '_api_token', None)
+        token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else 'Never'

-        response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
-        response["X-ArchiveBox-Auth-Expires"] = token_expiry
-        response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
-        response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
+        response['X-ArchiveBox-Auth-Method'] = getattr(request, '_api_auth_method', None) or 'None'
+        response['X-ArchiveBox-Auth-Expires'] = token_expiry
+        response['X-ArchiveBox-Auth-Token-Id'] = str(api_token.id) if api_token else 'None'
+        response['X-ArchiveBox-Auth-User-Id'] = request.user.pk if request.user.pk else 'None'
+        response['X-ArchiveBox-Auth-User-Username'] = request.user.username if request.user.pk else 'None'
+
+        # import ipdb; ipdb.set_trace()
+        # print('RESPONDING NOW', response)

        return response


 api = NinjaAPIWithIOCapture(
-    title="ArchiveBox API",
+    title='ArchiveBox API',
    description=html_description,
    version=VERSION,
    auth=API_AUTH_METHODS,
    urls_namespace="api-1",
    docs=Swagger(settings={"persistAuthorization": True}),
+    # docs_decorator=login_required,
+    # renderer=ORJSONRenderer(),
 )
 api = register_urls(api)
 urls = api.urls
@@ -112,16 +101,34 @@ def generic_exception_handler(request, err):
    if isinstance(err, (ObjectDoesNotExist, EmptyResultSet, PermissionDenied)):
        status = 404

-    print("".join(format_exception(err)))
+    print(''.join(format_exception(err)))

    return api.create_response(
        request,
        {
            "succeeded": False,
-            "message": f"{err.__class__.__name__}: {err}",
+            "message": f'{err.__class__.__name__}: {err}',
            "errors": [
-                "".join(format_exception(err)),
+                ''.join(format_exception(err)),
+                # or send simpler parent-only traceback:
+                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
            ],
        },
        status=status,
    )
+
+
+
+# import orjson
+# from ninja.renderers import BaseRenderer
+# class ORJSONRenderer(BaseRenderer):
+#     media_type = "application/json"
+#     def render(self, request, data, *, response_status):
+#         return {
+#             "success": True,
+#             "errors": [],
+#             "result": data,
+#             "stdout": ansi_to_html(stdout.getvalue().strip()),
+#             "stderr": ansi_to_html(stderr.getvalue().strip()),
+#         }
+#         return orjson.dumps(data)
@@ -1,28 +1,26 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from django.http import HttpRequest
+from typing import Optional

 from ninja import Router, Schema
+from django.utils import timezone
+from datetime import timedelta

-from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token, HeaderTokenAuth, BearerTokenAuth
+from archivebox.api.models import APIToken
+from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token


-router = Router(tags=["Authentication"], auth=None)
+router = Router(tags=['Authentication'], auth=None)


 class PasswordAuthSchema(Schema):
    """Schema for a /get_api_token request"""
-
-    username: str | None = None
-    password: str | None = None
+    username: Optional[str] = None
+    password: Optional[str] = None


-@router.post(
-    "/get_api_token",
-    auth=None,
-    summary="Generate an API token for a given username & password (or currently logged-in user)",
-)  # auth=None because they are not authed yet
-def get_api_token(request: HttpRequest, auth_data: PasswordAuthSchema):
+@router.post("/get_api_token", auth=None, summary='Generate an API token for a given username & password (or currently logged-in user)')             # auth=None because they are not authed yet
+def get_api_token(request, auth_data: PasswordAuthSchema):
    user = auth_using_password(
        username=auth_data.username,
        password=auth_data.password,
@@ -32,75 +30,24 @@ def get_api_token(request: HttpRequest, auth_data: PasswordAuthSchema):
    if user and user.is_superuser:
        api_token = get_or_create_api_token(user)
        assert api_token is not None, "Failed to create API token"
-        return {
-            "success": True,
-            "user_id": str(user.pk),
-            "username": user.username,
-            "token": api_token.token,
-            "expires": api_token.expires.isoformat() if api_token.expires else None,
-        }
-
+        return api_token.__json__()
+    
    return {"success": False, "errors": ["Invalid credentials"]}


+
 class TokenAuthSchema(Schema):
    """Schema for a /check_api_token request"""
-
    token: str


-@router.post(
-    "/check_api_token",
-    auth=None,
-    summary="Validate an API token to make sure its valid and non-expired",
-)  # auth=None because they are not authed yet
-def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
+@router.post("/check_api_token", auth=None, summary='Validate an API token to make sure its valid and non-expired')        # auth=None because they are not authed yet
+def check_api_token(request, token_data: TokenAuthSchema):
    user = auth_using_token(
        token=token_data.token,
        request=request,
    )
    if user:
        return {"success": True, "user_id": str(user.pk)}
-
+    
    return {"success": False, "user_id": None}
-
-
-@router.post(
-    "/browser_session",
-    auth=[HeaderTokenAuth(), BearerTokenAuth()],
-    summary="Exchange an administrator API key for a normal browser session",
-)
-def browser_session(request: HttpRequest):
-    from django.conf import settings
-    from django.contrib.auth import login
-    from django.http import JsonResponse
-    from django.utils import timezone
-    from ninja.errors import HttpError
-    from archivebox.core.routes_util import get_admin_base_url
-
-    user = request.auth
-    if not user.is_active or not user.is_superuser:
-        raise HttpError(403, "An active administrator is required")
-    login(request, user, backend="django.contrib.auth.backends.ModelBackend")
-    # A browser session must not outlive the key used to establish it.
-    token = request._api_token
-    lifetime = settings.SESSION_COOKIE_AGE
-    if token.expires:
-        lifetime = min(lifetime, max(0, int((token.expires - timezone.now()).total_seconds())))
-    request.session.set_expiry(lifetime)
-    request.session.save()
-    response = JsonResponse(
-        {
-            "admin_url": get_admin_base_url(request=request).rstrip("/") + "/admin/",
-            "cookie": {
-                "name": settings.SESSION_COOKIE_NAME,
-                "value": request.session.session_key,
-                "expires": request.session.get_expiry_date().timestamp(),
-                "secure": settings.SESSION_COOKIE_SECURE,
-            },
-        },
-    )
-    # The native client installs a host-only HttpOnly cookie on the admin origin;
-    # API and admin hosts can differ, so a Set-Cookie on the API host is insufficient.
-    response["Cache-Control"] = "no-store"
-    return response
@@ -1,329 +1,240 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

 import json
-from io import StringIO
-from typing import Any
+from typing import List, Dict, Any, Optional
 from enum import Enum

-from django.http import HttpRequest
-
 from ninja import Router, Schema
-from ninja.errors import HttpError
-from pydantic import Field

 from archivebox.misc.util import ansi_to_html
-from archivebox.core.models import SnapshotQuerySet
+from archivebox.config.common import ARCHIVING_CONFIG


+# from .auth import API_AUTH_METHODS
+
 # router for API that exposes archivebox cli subcommands as REST endpoints
-router = Router(tags=["ArchiveBox CLI Sub-Commands"])
+router = Router(tags=['ArchiveBox CLI Sub-Commands'])


 # Schemas

-JSONType = list[Any] | dict[str, Any] | bool | int | str | None
-FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
-
+JSONType = List[Any] | Dict[str, Any] | bool | int | str | None

 class CLICommandResponseSchema(Schema):
    success: bool
-    errors: list[str]
+    errors: List[str]
    result: JSONType
-    result_format: str = "str"
+    result_format: str = 'str'
    stdout: str
    stderr: str

+class FilterTypeChoices(str, Enum):
+    exact = 'exact'
+    substring = 'substring'
+    regex = 'regex'
+    domain = 'domain'
+    tag = 'tag'
+    timestamp = 'timestamp'

-FilterTypeChoices = Enum(
-    "FilterTypeChoices",
-    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
-    type=str,
-)
+class StatusChoices(str, Enum):
+    indexed = 'indexed'
+    archived = 'archived'
+    unarchived = 'unarchived'
+    present = 'present'
+    valid = 'valid'
+    invalid = 'invalid'
+    duplicate = 'duplicate'
+    orphaned = 'orphaned'
+    corrupted = 'corrupted'
+    unrecognized = 'unrecognized'


 class AddCommandSchema(Schema):
-    urls: list[str]
-    snapshot_ids: list[str] | None = None
+    urls: List[str]
    tag: str = ""
    depth: int = 0
-    max_urls: int = 0
-    crawl_max_size: int = 0
-    crawl_timeout: int = 0
-    snapshot_max_size: int = 0
    parser: str = "auto"
    plugins: str = ""
-    persona: str = "Default"
-    only_new: bool | None = None
-    update: bool = False
+    update: bool = not ARCHIVING_CONFIG.ONLY_NEW  # Default to the opposite of ARCHIVING_CONFIG.ONLY_NEW
    overwrite: bool = False
    index_only: bool = False

-
-class SnapshotFilterCommandSchema(Schema):
-    after: float | None = 0
-    before: float | None = None
-    filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
-    status: str | None = None
-    url__icontains: str | None = None
-    url__istartswith: str | None = None
-    tag: str | None = None
-    crawl_id: str | None = None
-    limit: int | None = None
-    sort: str | None = None
-    search: str | None = None
-
-
-class UpdateCommandSchema(SnapshotFilterCommandSchema):
-    resume: str | None = None
-    batch_size: int = 100
-    continuous: bool = False
+class UpdateCommandSchema(Schema):
+    resume: Optional[float] = 0
+    only_new: bool = ARCHIVING_CONFIG.ONLY_NEW
    index_only: bool = False
-    migrate_only: bool = False
-
+    overwrite: bool = False
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
+    status: Optional[StatusChoices] = StatusChoices.unarchived
+    filter_type: Optional[str] = FilterTypeChoices.substring
+    filter_patterns: Optional[List[str]] = ['https://example.com']
+    plugins: Optional[str] = ""

 class ScheduleCommandSchema(Schema):
-    import_path: str | None = None
+    import_path: Optional[str] = None
    add: bool = False
-    show: bool = False
-    foreground: bool = False
-    run_all: bool = False
-    quiet: bool = False
-    every: str | None = None
-    tag: str = ""
+    every: Optional[str] = None
+    tag: str = ''
    depth: int = 0
-    only_new: bool | None = None
-    update: bool = False
    overwrite: bool = False
+    update: bool = not ARCHIVING_CONFIG.ONLY_NEW
    clear: bool = False

-
-class ListCommandSchema(SnapshotFilterCommandSchema):
+class ListCommandSchema(Schema):
+    filter_patterns: Optional[List[str]] = ['https://example.com']
+    filter_type: str = FilterTypeChoices.substring
+    status: StatusChoices = StatusChoices.indexed
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
+    sort: str = 'bookmarked_at'
    as_json: bool = True
    as_html: bool = False
-    as_csv: str | None = "timestamp,url"
+    as_csv: str | None = 'timestamp,url'
    with_headers: bool = False

-
-class RemoveCommandSchema(SnapshotFilterCommandSchema):
+class RemoveCommandSchema(Schema):
+    delete: bool = True
+    after: Optional[float] = 0
+    before: Optional[float] = 999999999999999
    filter_type: str = FilterTypeChoices.exact
-    timeout: float = 60.0
+    filter_patterns: Optional[List[str]] = ['https://example.com']


-def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
-    kwargs = args.dict()
-    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
-    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
-    return kwargs


-@router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
-def cli_add(request: HttpRequest, args: AddCommandSchema):
+
+@router.post("/add", response=CLICommandResponseSchema, summary='archivebox add [args] [urls]')
+def cli_add(request, args: AddCommandSchema):
    from archivebox.cli.archivebox_add import add
-    from archivebox.misc.util import validate_url

-    config_overrides: dict[str, object] = {}
-    if args.only_new is not None:
-        config_overrides["ONLY_NEW"] = bool(args.only_new)
-    if args.update or args.overwrite:
-        config_overrides["ONLY_NEW"] = False
-    submitted_urls: str | list[str] = args.urls
-    if len(args.urls) == 1:
-        try:
-            validate_url(args.urls[0])
-        except ValueError:
-            submitted_urls = args.urls[0]
-    else:
-        submitted_urls = []
-        for url in args.urls:
-            try:
-                submitted_urls.append(validate_url(url))
-            except ValueError:
-                continue
-        if not submitted_urls:
-            raise HttpError(400, "No valid URLs were submitted")
-    crawl, snapshots = add(
-        urls=submitted_urls,
-        snapshot_ids=args.snapshot_ids,
+    result = add(
+        urls=args.urls,
        tag=args.tag,
        depth=args.depth,
-        max_urls=args.max_urls,
-        crawl_max_size=args.crawl_max_size,
-        crawl_timeout=args.crawl_timeout,
-        snapshot_max_size=args.snapshot_max_size,
+        update=args.update,
        index_only=args.index_only,
+        overwrite=args.overwrite,
        plugins=args.plugins,
-        persona=args.persona,
        parser=args.parser,
        bg=True,  # Always run in background for API calls
        created_by_id=request.user.pk,
-        config=config_overrides or None,
    )

-    snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
-    result_payload = {
-        "crawl_id": str(crawl.id),
-        "num_snapshots": len(snapshot_ids),
-        "snapshot_ids": snapshot_ids,
-        "queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
-    }
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-
-    return {
-        "success": True,
-        "errors": [],
-        "result": result_payload,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
-def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
-    from archivebox.core.snapshot_status import normalize_snapshot_status
-
-    try:
-        status = normalize_snapshot_status(args.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
-    update_kwargs["status"] = status
-    update_kwargs["stop_daemon_stack"] = False
-
-    is_filtered_update = any(
-        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
-    )
-    matched_snapshot_ids = []
-    if is_filtered_update:
-        matched_snapshot_ids = [
-            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
-        ]
-
-    update(**update_kwargs)
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
-    return {
-        "success": True,
-        "errors": [],
-        "result": {
-            "matched_count": len(matched_snapshot_ids),
-            "snapshot_ids": matched_snapshot_ids,
-        }
-        if is_filtered_update
-        else None,
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
-    }
-
-
-@router.post("/schedule", response=CLICommandResponseSchema, summary="archivebox schedule [args] [import_path]")
-def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
-    from archivebox.cli.archivebox_schedule import schedule
-
-    config_overrides: dict[str, object] = {}
-    if args.only_new is not None:
-        config_overrides["ONLY_NEW"] = bool(args.only_new)
-    if args.update or args.overwrite:
-        config_overrides["ONLY_NEW"] = False
-    result = schedule(
-        import_path=args.import_path,
-        add=args.add,
-        show=args.show,
-        foreground=args.foreground,
-        run_all=args.run_all,
-        quiet=args.quiet,
-        clear=args.clear,
-        every=args.every,
-        tag=args.tag,
-        depth=args.depth,
-        config=config_overrides or None,
-    )
-
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
    return {
        "success": True,
        "errors": [],
        "result": result,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
    }


-@router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
-def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
+@router.post("/update", response=CLICommandResponseSchema, summary='archivebox update [args] [filter_patterns]')
+def cli_update(request, args: UpdateCommandSchema):
+    from archivebox.cli.archivebox_update import update
+    
+    result = update(
+        resume=args.resume,
+        only_new=args.only_new,
+        index_only=args.index_only,
+        overwrite=args.overwrite,
+        before=args.before,
+        after=args.after,
+        status=args.status,
+        filter_type=args.filter_type,
+        filter_patterns=args.filter_patterns,
+        plugins=args.plugins,
+    )
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }

-    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
-    as_json = search_kwargs.pop("as_json")
-    as_html = search_kwargs.pop("as_html")
-    as_csv = search_kwargs.pop("as_csv")
-    with_headers = search_kwargs.pop("with_headers")
-    try:
-        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err

-    result_format = "txt"
-    if as_json:
+@router.post("/schedule", response=CLICommandResponseSchema, summary='archivebox schedule [args] [import_path]')
+def cli_schedule(request, args: ScheduleCommandSchema):
+    from archivebox.cli.archivebox_schedule import schedule
+    
+    result = schedule(
+        import_path=args.import_path,
+        add=args.add,
+        show=args.show,
+        clear=args.clear,
+        every=args.every,
+        tag=args.tag,
+        depth=args.depth,
+        overwrite=args.overwrite,
+        update=args.update,
+    )
+
+    return {
+        "success": True,
+        "errors": [],
+        "result": result,
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
+    }
+
+
+
+@router.post("/search", response=CLICommandResponseSchema, summary='archivebox search [args] [filter_patterns]')
+def cli_search(request, args: ListCommandSchema):
+    from archivebox.cli.archivebox_search import search
+    
+    result = search(
+        filter_patterns=args.filter_patterns,
+        filter_type=args.filter_type,
+        status=args.status,
+        after=args.after,
+        before=args.before,
+        sort=args.sort,
+        csv=args.as_csv,
+        json=args.as_json,
+        html=args.as_html,
+        with_headers=args.with_headers,
+    )
+
+    result_format = 'txt'
+    if args.as_json:
        result_format = "json"
-        result = [
-            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
-            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
-        ]
-    elif as_html:
+        result = json.loads(result)
+    elif args.as_html:
        result_format = "html"
-        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
-    elif as_csv:
+    elif args.as_csv:
        result_format = "csv"
-        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
-        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
-        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
-    else:
-        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))

-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
    return {
        "success": True,
        "errors": [],
        "result": result,
        "result_format": result_format,
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
    }
+    


-@router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
-def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
+@router.post("/remove", response=CLICommandResponseSchema, summary='archivebox remove [args] [filter_patterns]')
+def cli_remove(request, args: RemoveCommandSchema):
    from archivebox.cli.archivebox_remove import remove
-    from archivebox.core.models import Snapshot
-
-    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
-    timeout_arg = remove_kwargs.pop("timeout")
-    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
-    try:
-        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
+    
    result = remove(
-        yes=True,  # no way to interactively ask for confirmation via API, so we force yes
-        snapshots=snapshots_to_remove,
-        timeout=timeout,
+        yes=True,            # no way to interactively ask for confirmation via API, so we force yes
+        delete=args.delete,
+        before=args.before,
+        after=args.after,
+        filter_type=args.filter_type,
+        filter_patterns=args.filter_patterns,
    )
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
    return {
-        "success": bool(result["success"]),
-        "errors": [str(result["error"])] if result["error"] else [],
+        "success": True,
+        "errors": [],
        "result": result,
-        "result_format": "json",
-        "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
-        "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
+        "stdout": ansi_to_html(request.stdout.getvalue().strip()),
+        "stderr": ansi_to_html(request.stderr.getvalue().strip()),
    }
+    
@@ -1,36 +1,24 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

-from pathlib import Path
 from uuid import UUID
+from typing import List
 from datetime import datetime
-from django.http import FileResponse, HttpRequest
-from django.shortcuts import redirect
-from django.utils import timezone

+from django.db.models import Q
 from django.contrib.auth import get_user_model
-from django.contrib.auth.models import User

 from ninja import Router, Schema
-from ninja.errors import HttpError

 from archivebox.core.models import Snapshot
-from archivebox.core.permissions import (
-    PERMISSIONS_PUBLIC,
-    PERMISSIONS_UNLISTED,
-    is_admin_user,
-    normalize_permissions,
-)
-from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
-from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url

-from .auth import API_AUTH_METHODS, authenticated_user_from_request
+from .auth import API_AUTH_METHODS

-router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
+router = Router(tags=['Crawl Models'], auth=API_AUTH_METHODS)


 class CrawlSchema(Schema):
-    TYPE: str = "crawls.models.Crawl"
+    TYPE: str = 'crawls.models.Crawl'

    id: UUID

@@ -41,238 +29,53 @@ class CrawlSchema(Schema):

    status: str
    retry_at: datetime | None
-    is_paused: bool

    urls: str
+    extractor: str
    max_depth: int
    tags_str: str
    config: dict

+    # snapshots: List[SnapshotSchema]
+
    @staticmethod
    def resolve_created_by_id(obj):
        return str(obj.created_by_id)

    @staticmethod
    def resolve_created_by_username(obj):
-        user_model = get_user_model()
-        user = user_model.objects.get(id=obj.created_by_id)
-        username = user.username
-        return username if isinstance(username, str) else str(user)
-
-    @staticmethod
-    def resolve_config(obj):
-        # Redact credential values so REST responses can never leak the raw
-        # token/secret/api-key that the operator stored in Crawl.config.
-        from archivebox.config.common import redact_sensitive_config
-
-        return redact_sensitive_config(obj.config)
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username

    @staticmethod
    def resolve_snapshots(obj, context):
-        if bool(context["request"].__dict__.get("with_snapshots", False)):
+        if context['request'].with_snapshots:
            return obj.snapshot_set.all().distinct()
        return Snapshot.objects.none()


-class CrawlUpdateSchema(Schema):
-    action: str | None = None
-    status: str | None = None
-    retry_at: datetime | None = None
-    tags: list[str] | None = None
-    tags_str: str | None = None
-
-
-class CrawlCreateSchema(Schema):
-    urls: list[str]
-    max_depth: int = 0
-    tags: list[str] | None = None
-    tags_str: str = ""
-    label: str = ""
-    notes: str = ""
-    config: dict = {}
-
-
-class CrawlDeleteResponseSchema(Schema):
-    success: bool
-    crawl_id: str
-    deleted_count: int
-    deleted_snapshots: int
-
-
-def normalize_tag_list(tags: list[str] | None = None, tags_str: str = "") -> list[str]:
-    if tags is not None:
-        return [tag.strip() for tag in tags if tag and tag.strip()]
-    return [tag.strip() for tag in tags_str.split(",") if tag.strip()]
-
-
-@router.get("/crawls", response=list[CrawlSchema], url_name="get_crawls")
-def get_crawls(request: HttpRequest):
+@router.get("/crawls", response=List[CrawlSchema], url_name="get_crawls")
+def get_crawls(request):
    return Crawl.objects.all().distinct()

-
-@router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
-def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
-    try:
-        urls = [validate_url(url) for url in data.urls if url and url.strip()]
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    if not urls:
-        raise HttpError(400, "At least one URL is required")
-    if data.max_depth not in (0, 1, 2, 3, 4):
-        raise HttpError(400, "max_depth must be between 0 and 4")
-
-    tags = normalize_tag_list(data.tags, data.tags_str)
-    config = dict(data.config or {})
-    config.setdefault("PARSER", "url_list")
-    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
-    crawl = Crawl.objects.create(
-        urls="\n".join(urls),
-        max_depth=data.max_depth,
-        tags_str=",".join(tags),
-        label=data.label,
-        notes=data.notes,
-        config=config,
-        status=Crawl.StatusChoices.QUEUED,
-        retry_at=timezone.now(),
-        created_by=request.user if isinstance(request.user, User) else None,
-    )
-    return crawl
-
-
-def get_crawl_by_ref(crawl_id: str):
-    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
-
-
-@router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
-def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
+@router.get("/crawl/{crawl_id}", response=CrawlSchema | str, url_name="get_crawl")
+def get_crawl(request, crawl_id: str, as_rss: bool=False, with_snapshots: bool=False, with_archiveresults: bool=False):
    """Get a specific Crawl by id."""
-    setattr(request, "with_snapshots", with_snapshots)
-    setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = get_crawl_by_ref(crawl_id)
-
+    request.with_snapshots = with_snapshots
+    request.with_archiveresults = with_archiveresults
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
+    
    if crawl and as_rss:
-        query = request.GET.copy()
-        query.pop("as_rss", None)
-        query["crawl_id"] = str(crawl.id)
-        return redirect(f"/api/v1/core/snapshots.rss?{query.urlencode()}")
-
+        # return snapshots as XML rss feed
+        urls = [
+            {'url': snapshot.url, 'title': snapshot.title, 'bookmarked_at': snapshot.bookmarked_at, 'tags': snapshot.tags_str}
+            for snapshot in crawl.snapshot_set.all()
+        ]
+        xml = '<rss version="2.0"><channel>'
+        for url in urls:
+            xml += f'<item><url>{url["url"]}</url><title>{url["title"]}</title><bookmarked_at>{url["bookmarked_at"]}</bookmarked_at><tags>{url["tags"]}</tags></item>'
+        xml += '</channel></rss>'
+        return xml
+    
    return crawl

-
-def crawl_file(request: HttpRequest, crawl_id: str, path: str):
-    # Try to resolve the crawl first; if it doesn't exist, return 404.
-    try:
-        crawl = get_crawl_by_ref(crawl_id)
-    except Crawl.DoesNotExist:
-        raise HttpError(404, "Crawl not found")
-
-    user = authenticated_user_from_request(request)
-
-    # Gate access using the same model as SnapshotView/can_view_snapshot:
-    # admins always pass; owners can see their own crawls; otherwise the crawl
-    # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
-    if not is_admin_user(request):
-        permissions = normalize_permissions(crawl.permissions)
-        is_owner = bool(user and crawl.created_by_id == user.id)
-        if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
-            raise HttpError(404, "Crawl not found")
-
-    crawl_root = Path(crawl.output_dir).resolve()
-    file_path = (crawl_root / path).resolve()
-    if not file_path.is_file() or crawl_root not in file_path.parents:
-        raise HttpError(404, "Crawl file not found")
-
-    if path == "chrome_screencast/latest.jpg" and request.GET.get("after"):
-        try:
-            started_at = float(request.GET["after"])
-        except ValueError:
-            raise HttpError(400, "Invalid screencast start time")
-        if file_path.stat().st_mtime < started_at:
-            raise HttpError(404, "No frame for this capture yet")
-
-    response = FileResponse(file_path.open("rb"))
-    response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
-    response["Pragma"] = "no-cache"
-    response["Expires"] = "0"
-    response["X-Content-Type-Options"] = "nosniff"
-    return response
-
-
-@router.get("/crawl/{crawl_id}/files/{filename}", auth=None, url_name="crawl_file_root")
-def crawl_file_root(request: HttpRequest, crawl_id: str, filename: str):
-    return crawl_file(request, crawl_id, filename)
-
-
-@router.get("/crawl/{crawl_id}/files/{folder}/{filename}", auth=None, url_name="crawl_file_nested_1")
-def crawl_file_nested_1(request: HttpRequest, crawl_id: str, folder: str, filename: str):
-    return crawl_file(request, crawl_id, f"{folder}/{filename}")
-
-
-@router.get("/crawl/{crawl_id}/files/{folder}/{subfolder}/{filename}", auth=None, url_name="crawl_file_nested_2")
-def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfolder: str, filename: str):
-    return crawl_file(request, crawl_id, f"{folder}/{subfolder}/{filename}")
-
-
-@router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
-def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
-    """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = get_crawl_by_ref(crawl_id)
-    payload = data.dict(exclude_unset=True)
-    update_fields = ["modified_at"]
-
-    action = payload.pop("action", None)
-    if action:
-        if action == "pause":
-            crawl.pause()
-            return crawl
-        if action in ("resume", "unpause"):
-            crawl.resume()
-            return crawl
-        if action == "cancel":
-            crawl.cancel()
-            return crawl
-        raise HttpError(400, f"Invalid action: {action}")
-
-    tags = payload.pop("tags", None)
-    tags_str = payload.pop("tags_str", None)
-    if tags is not None or tags_str is not None:
-        crawl.tags_str = ",".join(normalize_tag_list(tags, tags_str or ""))
-        update_fields.append("tags_str")
-
-    if "status" in payload:
-        if payload["status"] not in Crawl.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        if payload["status"] == Crawl.StatusChoices.SEALED:
-            crawl.cancel()
-            return crawl
-        crawl.status = payload["status"]
-        update_fields.append("status")
-
-    if "retry_at" in payload:
-        crawl.retry_at = payload["retry_at"]
-        update_fields.append("retry_at")
-
-    crawl.save(update_fields=update_fields)
-    return crawl
-
-
-@router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
-def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = get_crawl_by_ref(crawl_id)
-    crawl_id_str = str(crawl.id)
-    crawl.cancel()
-
-    from archivebox.crawls.locks import crawl_lifecycle_lock
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        crawl = get_crawl_by_ref(crawl_id_str)
-        snapshot_count = crawl.snapshot_set.count()
-        deleted_count, _ = crawl.delete()
-    return {
-        "success": True,
-        "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
-        "deleted_snapshots": snapshot_count,
-    }
@@ -1,29 +1,25 @@
-__package__ = "archivebox.api"
+__package__ = 'archivebox.api'

 from uuid import UUID
-from typing import Annotated
+from typing import List, Optional
 from datetime import datetime

-from django.http import HttpRequest
-
-from ninja import FilterLookup, FilterSchema, Query, Router, Schema
+from ninja import Router, Schema, FilterSchema, Field, Query
 from ninja.pagination import paginate

 from archivebox.api.v1_core import CustomPagination


-router = Router(tags=["Machine and Dependencies"])
+router = Router(tags=['Machine and Dependencies'])


 # ============================================================================
 # Machine Schemas
 # ============================================================================

-
 class MachineSchema(Schema):
    """Schema for Machine model."""
-
-    TYPE: str = "machine.Machine"
+    TYPE: str = 'machine.Machine'
    id: UUID
    created_at: datetime
    modified_at: datetime
@@ -45,24 +41,25 @@ class MachineSchema(Schema):


 class MachineFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup("id__startswith")] = None
-    hostname: Annotated[str | None, FilterLookup("hostname__icontains")] = None
-    os_platform: Annotated[str | None, FilterLookup("os_platform__icontains")] = None
-    os_arch: Annotated[str | None, FilterLookup("os_arch")] = None
-    hw_in_docker: Annotated[bool | None, FilterLookup("hw_in_docker")] = None
-    hw_in_vm: Annotated[bool | None, FilterLookup("hw_in_vm")] = None
-    bin_providers: Annotated[str | None, FilterLookup("bin_providers__icontains")] = None
+    id: Optional[str] = Field(None, q='id__startswith')
+    hostname: Optional[str] = Field(None, q='hostname__icontains')
+    os_platform: Optional[str] = Field(None, q='os_platform__icontains')
+    os_arch: Optional[str] = Field(None, q='os_arch')
+    hw_in_docker: Optional[bool] = Field(None, q='hw_in_docker')
+    hw_in_vm: Optional[bool] = Field(None, q='hw_in_vm')
+
+
+# ============================================================================
+    bin_providers: Optional[str] = Field(None, q='bin_providers__icontains')


 # ============================================================================
 # Binary Schemas
 # ============================================================================

-
 class BinarySchema(Schema):
    """Schema for Binary model."""
-
-    TYPE: str = "machine.Binary"
+    TYPE: str = 'machine.Binary'
    id: UUID
    created_at: datetime
    modified_at: datetime
@@ -89,45 +86,41 @@ class BinarySchema(Schema):


 class BinaryFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup("id__startswith")] = None
-    name: Annotated[str | None, FilterLookup("name__icontains")] = None
-    binprovider: Annotated[str | None, FilterLookup("binprovider")] = None
-    status: Annotated[str | None, FilterLookup("status")] = None
-    machine_id: Annotated[str | None, FilterLookup("machine_id__startswith")] = None
-    version: Annotated[str | None, FilterLookup("version__icontains")] = None
+    id: Optional[str] = Field(None, q='id__startswith')
+    name: Optional[str] = Field(None, q='name__icontains')
+    binprovider: Optional[str] = Field(None, q='binprovider')
+    status: Optional[str] = Field(None, q='status')
+    machine_id: Optional[str] = Field(None, q='machine_id__startswith')
+    version: Optional[str] = Field(None, q='version__icontains')


 # ============================================================================
 # Machine Endpoints
 # ============================================================================

-
-@router.get("/machines", response=list[MachineSchema], url_name="get_machines")
+@router.get("/machines", response=List[MachineSchema], url_name="get_machines")
@paginate(CustomPagination)
-def get_machines(request: HttpRequest, filters: Query[MachineFilterSchema]):
+def get_machines(request, filters: MachineFilterSchema = Query(...)):
    """List all machines."""
    from archivebox.machine.models import Machine
-
    return filters.filter(Machine.objects.all()).distinct()


-@router.get("/machine/current", response=MachineSchema, url_name="get_current_machine")
-def get_current_machine(request: HttpRequest):
-    """Get the current machine."""
-    from archivebox.machine.models import Machine
-
-    return Machine.current()
-
-
@router.get("/machine/{machine_id}", response=MachineSchema, url_name="get_machine")
-def get_machine(request: HttpRequest, machine_id: str):
+def get_machine(request, machine_id: str):
    """Get a specific machine by ID."""
    from archivebox.machine.models import Machine
    from django.db.models import Q
-
    return Machine.objects.get(Q(id__startswith=machine_id) | Q(hostname__iexact=machine_id))


+@router.get("/machine/current", response=MachineSchema, url_name="get_current_machine")
+def get_current_machine(request):
+    """Get the current machine."""
+    from archivebox.machine.models import Machine
+    return Machine.current()
+
+
 # ============================================================================


@@ -135,27 +128,23 @@ def get_machine(request: HttpRequest, machine_id: str):
 # Binary Endpoints
 # ============================================================================

-
-@router.get("/binaries", response=list[BinarySchema], url_name="get_binaries")
+@router.get("/binaries", response=List[BinarySchema], url_name="get_binaries")
@paginate(CustomPagination)
-def get_binaries(request: HttpRequest, filters: Query[BinaryFilterSchema]):
+def get_binaries(request, filters: BinaryFilterSchema = Query(...)):
    """List all binaries."""
    from archivebox.machine.models import Binary
-
-    return filters.filter(Binary.objects.all().select_related("machine")).distinct()
+    return filters.filter(Binary.objects.all().select_related('machine', 'dependency')).distinct()


@router.get("/binary/{binary_id}", response=BinarySchema, url_name="get_binary")
-def get_binary(request: HttpRequest, binary_id: str):
+def get_binary(request, binary_id: str):
    """Get a specific binary by ID."""
    from archivebox.machine.models import Binary
-
-    return Binary.objects.select_related("machine").get(id__startswith=binary_id)
+    return Binary.objects.select_related('machine', 'dependency').get(id__startswith=binary_id)


-@router.get("/binary/by-name/{name}", response=list[BinarySchema], url_name="get_binaries_by_name")
-def get_binaries_by_name(request: HttpRequest, name: str):
+@router.get("/binary/by-name/{name}", response=List[BinarySchema], url_name="get_binaries_by_name")
+def get_binaries_by_name(request, name: str):
    """Get all binaries with the given name."""
    from archivebox.machine.models import Binary
-
-    return list(Binary.objects.filter(name__iexact=name).select_related("machine"))
+    return list(Binary.objects.filter(name__iexact=name).select_related('machine', 'dependency'))
@@ -1,176 +0,0 @@
-__package__ = "archivebox.api"
-
-import json
-from datetime import datetime
-from typing import Any, Literal
-from uuid import UUID
-
-from django.core.exceptions import ValidationError
-from django.db.models import Q
-from django.http import HttpRequest
-from ninja import Router, Schema
-from ninja.errors import HttpError
-from ninja.pagination import paginate
-from pydantic import Field
-
-from archivebox.api.v1_core import CustomPagination
-from archivebox.personas.importers import validate_persona_name
-from archivebox.personas.models import Persona
-
-
-router = Router(tags=["Personas"])
-
-
-class PersonaBrowserSettingsSchema(Schema):
-    user_agent: str = ""
-    viewport_size: str = ""
-    viewport_device_scale_factor: float | None = None
-    color_scheme: Literal["", "light", "dark"] = ""
-    language: str = ""
-    timezone: str = ""
-    geolocation: dict[str, Any] | None = None
-
-
-class PersonaSyncSchema(Schema):
-    extension_persona_id: str
-    name: str
-    settings: PersonaBrowserSettingsSchema = Field(default_factory=PersonaBrowserSettingsSchema)
-    cookies_txt: str = ""
-    auth_json: dict[str, Any] = Field(default_factory=dict)
-
-
-class PersonaSchema(Schema):
-    TYPE: str = "personas.models.Persona"
-    id: UUID
-    name: str
-    created_at: datetime
-    created_by_id: str
-    created_by_username: str
-    config: dict[str, Any] | None
-
-    @staticmethod
-    def resolve_created_by_id(obj):
-        return str(obj.created_by.pk)
-
-    @staticmethod
-    def resolve_created_by_username(obj) -> str:
-        return obj.created_by.username
-
-    @staticmethod
-    def resolve_config(obj):
-        # Redact credential values so REST responses don't leak the raw
-        # token/secret/api-key the operator stored in Persona.config.
-        from archivebox.config.common import redact_sensitive_config
-
-        return redact_sensitive_config(obj.config)
-
-
-class PersonaSyncResponseSchema(Schema):
-    success: bool
-    created: bool
-    persona: PersonaSchema
-    cookies_file_written: bool
-    auth_file_written: bool
-
-
-def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrowserSettingsSchema) -> dict[str, Any]:
-    config: dict[str, Any] = {
-        "BROWSER_EXTENSION_PERSONA_ID": extension_persona_id,
-        "BROWSER_EXTENSION_SYNCED_AT": datetime.utcnow().isoformat() + "Z",
-    }
-
-    if settings.user_agent:
-        config.update(
-            {
-                "USER_AGENT": settings.user_agent,
-                "CHROME_USER_AGENT": settings.user_agent,
-                "WGET_USER_AGENT": settings.user_agent,
-                "CURL_USER_AGENT": settings.user_agent,
-            },
-        )
-    if settings.viewport_size:
-        config.update(
-            {
-                "RESOLUTION": settings.viewport_size,
-                "CHROME_RESOLUTION": settings.viewport_size,
-            },
-        )
-    if settings.viewport_device_scale_factor is not None:
-        config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
-    if "color_scheme" in settings.model_fields_set:
-        config["BROWSER_COLOR_SCHEME"] = settings.color_scheme
-    if settings.language:
-        config["BROWSER_LANGUAGE"] = settings.language
-    if settings.timezone:
-        config["BROWSER_TIMEZONE"] = settings.timezone
-    if settings.geolocation:
-        config["BROWSER_GEOLOCATION"] = settings.geolocation
-
-    return config
-
-
-def find_persona(extension_persona_id: str, name: str) -> Persona | None:
-    named = Persona.find_named(name)
-    return (
-        Persona.objects.filter(Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(pk=named.pk if named else None))
-        .order_by("created_at")
-        .first()
-    )
-
-
-@router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
-@paginate(CustomPagination)
-def get_personas(request: HttpRequest):
-    """List personas available on this ArchiveBox server."""
-    return Persona.objects.all().order_by("name")
-
-
-@router.post("/sync", response=PersonaSyncResponseSchema, url_name="sync_persona")
-def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
-    """
-    Create or update a Persona from a browser extension profile export.
-
-    The extension sends browser settings plus portable auth artifacts. The server
-    keeps browser override settings in Persona.config and writes cookies.txt /
-    auth.json into the persona directory for extractors to consume.
-    """
-    name = payload.name.strip()
-    is_valid, error_message = validate_persona_name(name)
-    if not is_valid:
-        raise ValueError(error_message)
-
-    try:
-        persona = find_persona(payload.extension_persona_id, name)
-        (persona or Persona(name=name)).validate_name(persona.name if persona else name)
-    except ValidationError as err:
-        raise HttpError(409, "; ".join(err.messages)) from err
-    created = persona is None
-    if persona is None:
-        persona = Persona(name=name)
-        if request.user.is_authenticated:
-            persona.created_by = request.user
-
-    persona.config = {
-        **(persona.config or {}),
-        **browser_settings_to_config(payload.extension_persona_id, payload.settings),
-    }
-    persona.save()
-    persona.ensure_dirs()
-
-    cookies_written = False
-    if "cookies_txt" in payload.model_fields_set:
-        (persona.path / "cookies.txt").write_text(payload.cookies_txt)
-        cookies_written = True
-
-    auth_written = False
-    if "auth_json" in payload.model_fields_set:
-        (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
-        auth_written = True
-
-    return {
-        "success": True,
-        "created": created,
-        "persona": persona,
-        "cookies_file_written": cookies_written,
-        "auth_file_written": auth_written,
-    }
@@ -0,0 +1,160 @@
+__package__ = 'archivebox.api'
+
+from uuid import UUID
+from typing import List, Any
+from datetime import datetime
+
+from ninja import Router, Schema
+
+
+router = Router(tags=['Workers and Tasks'])
+
+
+class QueueItemSchema(Schema):
+    """Schema for a single item in a worker's queue."""
+    TYPE: str
+    id: UUID
+    status: str
+    retry_at: datetime | None
+    created_at: datetime
+    modified_at: datetime
+    description: str
+
+    @staticmethod
+    def resolve_TYPE(obj) -> str:
+        return f'{obj._meta.app_label}.{obj._meta.model_name}'
+
+    @staticmethod
+    def resolve_description(obj) -> str:
+        return str(obj)
+
+
+class WorkerSchema(Schema):
+    """Schema for a Worker type."""
+    name: str
+    model: str
+    max_tick_time: int
+    max_concurrent_tasks: int
+    poll_interval: float
+    idle_timeout: int
+    running_count: int
+    running_workers: List[dict[str, Any]]
+    queue_count: int
+    queue: List[QueueItemSchema]
+
+    @staticmethod
+    def resolve_model(obj) -> str:
+        Model = obj.get_model()
+        return f'{Model._meta.app_label}.{Model._meta.model_name}'
+
+    @staticmethod
+    def resolve_max_tick_time(obj) -> int:
+        return obj.MAX_TICK_TIME
+
+    @staticmethod
+    def resolve_max_concurrent_tasks(obj) -> int:
+        return obj.MAX_CONCURRENT_TASKS
+
+    @staticmethod
+    def resolve_poll_interval(obj) -> float:
+        return obj.POLL_INTERVAL
+
+    @staticmethod
+    def resolve_idle_timeout(obj) -> int:
+        return obj.IDLE_TIMEOUT
+
+    @staticmethod
+    def resolve_running_count(obj) -> int:
+        return len(obj.get_running_workers())
+
+    @staticmethod
+    def resolve_running_workers(obj) -> List[dict[str, Any]]:
+        return obj.get_running_workers()
+
+    @staticmethod
+    def resolve_queue_count(obj) -> int:
+        return obj.get_queue().count()
+
+    @staticmethod
+    def resolve_queue(obj) -> List[QueueItemSchema]:
+        return list(obj.get_queue()[:50])  # Limit to 50 items
+
+
+class OrchestratorSchema(Schema):
+    """Schema for the Orchestrator."""
+    is_running: bool
+    poll_interval: float
+    idle_timeout: int
+    max_workers_per_type: int
+    max_total_workers: int
+    total_worker_count: int
+    workers: List[WorkerSchema]
+
+
+@router.get("/orchestrator", response=OrchestratorSchema, url_name="get_orchestrator")
+def get_orchestrator(request):
+    """Get the orchestrator status and all worker queues."""
+    from archivebox.workers.orchestrator import Orchestrator
+    from archivebox.workers.worker import CrawlWorker, SnapshotWorker, ArchiveResultWorker
+
+    orchestrator = Orchestrator()
+
+    # Create temporary worker instances to query their queues
+    workers = [
+        CrawlWorker(worker_id=-1),
+        SnapshotWorker(worker_id=-1),
+        ArchiveResultWorker(worker_id=-1),
+    ]
+
+    return {
+        'is_running': orchestrator.is_running(),
+        'poll_interval': orchestrator.POLL_INTERVAL,
+        'idle_timeout': orchestrator.IDLE_TIMEOUT,
+        'max_workers_per_type': orchestrator.MAX_WORKERS_PER_TYPE,
+        'max_total_workers': orchestrator.MAX_TOTAL_WORKERS,
+        'total_worker_count': orchestrator.get_total_worker_count(),
+        'workers': workers,
+    }
+
+
+@router.get("/workers", response=List[WorkerSchema], url_name="get_workers")
+def get_workers(request):
+    """List all worker types and their current status."""
+    from archivebox.workers.worker import CrawlWorker, SnapshotWorker, ArchiveResultWorker
+
+    # Create temporary instances to query their queues
+    return [
+        CrawlWorker(worker_id=-1),
+        SnapshotWorker(worker_id=-1),
+        ArchiveResultWorker(worker_id=-1),
+    ]
+
+
+@router.get("/worker/{worker_name}", response=WorkerSchema, url_name="get_worker")
+def get_worker(request, worker_name: str):
+    """Get status and queue for a specific worker type."""
+    from archivebox.workers.worker import WORKER_TYPES
+
+    if worker_name not in WORKER_TYPES:
+        from ninja.errors import HttpError
+        raise HttpError(404, f"Unknown worker type: {worker_name}. Valid types: {list(WORKER_TYPES.keys())}")
+
+    WorkerClass = WORKER_TYPES[worker_name]
+    return WorkerClass(worker_id=-1)
+
+
+@router.get("/worker/{worker_name}/queue", response=List[QueueItemSchema], url_name="get_worker_queue")
+def get_worker_queue(request, worker_name: str, limit: int = 100):
+    """Get the current queue for a specific worker type."""
+    from archivebox.workers.worker import WORKER_TYPES
+
+    if worker_name not in WORKER_TYPES:
+        from ninja.errors import HttpError
+        raise HttpError(404, f"Unknown worker type: {worker_name}. Valid types: {list(WORKER_TYPES.keys())}")
+
+    WorkerClass = WORKER_TYPES[worker_name]
+    worker = WorkerClass(worker_id=-1)
+    return list(worker.get_queue()[:limit])
+
+
+# Progress endpoint moved to core.views.live_progress_view for simplicity
@@ -1,60 +0,0 @@
-__package__ = "archivebox.api"
-
-from typing import Any
-from collections.abc import Callable
-import logging
-
-from django.db import transaction
-from django.db.models.signals import pre_delete
-from signal_webhooks.handlers import sync_task_handler
-
-
-logger = logging.getLogger(__name__)
-
-
-def load_deferred_fields_for_delete_webhook(sender, instance, using, **kwargs) -> None:
-    """Materialize deferred fields while DELETE webhook rows still exist.
-
-    Admin lists defer large fields, but signal_webhooks serializes all concrete
-    fields in post_delete. Loading one then would query an already-deleted row
-    and drop the event. Hydrate only deferred fields on webhook-enabled models
-    here, preserving the normal list-query optimization.
-    """
-    from signal_webhooks.handlers import find_hook_handler
-    from signal_webhooks.utils import reference_for_model
-
-    ref = reference_for_model(type(instance))
-    deferred_fields = instance.get_deferred_fields()
-    if not deferred_fields or find_hook_handler(ref, "DELETE") is None:
-        return
-
-    instance.refresh_from_db(using=using, fields=deferred_fields)
-
-
-def register_delete_webhook_field_loader() -> None:
-    pre_delete.connect(
-        load_deferred_fields_for_delete_webhook,
-        dispatch_uid="archivebox.load_deferred_fields_for_delete_webhook",
-        weak=False,
-    )
-
-
-def warning_error_handler(hook: Any, error: Exception | None) -> None:
-    if error is not None:
-        logger.warning("Outbound webhook %r failed: %s", hook.name, error)
-        return
-
-    logger.warning("Outbound webhook %r returned a non-success response.", hook.name)
-
-
-def transaction_on_commit_task_handler(hook: Callable[..., None], **kwargs: Any) -> None:
-    def run_webhook() -> None:
-        try:
-            sync_task_handler(hook, **kwargs)
-        except Exception:
-            logger.warning("Outbound webhook failed after transaction commit.", exc_info=True)
-
-    try:
-        transaction.on_commit(run_webhook)
-    except Exception:
-        logger.warning("Could not schedule outbound webhook after transaction commit.", exc_info=True)
@@ -1 +1 @@
-__package__ = "archivebox.base_models"
+__package__ = 'archivebox.base_models'
@@ -0,0 +1,7 @@
+# from django.apps import AppConfig
+
+
+# class BaseModelsConfig(AppConfig):
+#     default_auto_field = 'django.db.models.BigAutoField'
+
+#     name = 'base_models'
@@ -1,19 +1,18 @@
 """Base models using UUIDv7 for all id fields."""

-__package__ = "archivebox.base_models"
+__package__ = 'archivebox.base_models'

+import io
+import csv
 import json
-import shutil
-from typing import Any
-
-from archivebox.uuid_compat import CompactUUIDField, uuid7
+from uuid import UUID
+from archivebox.uuid_compat import uuid7
+from typing import Any, Iterable, ClassVar
 from pathlib import Path

-from django.core.exceptions import FieldDoesNotExist
+from django.contrib import admin
 from django.db import models
 from django.db.models import F
-from django.db import transaction
-from django.db.models.signals import pre_delete
 from django.utils import timezone
 from django.contrib.auth import get_user_model
 from django.urls import reverse_lazy
@@ -21,150 +20,28 @@ from django.conf import settings

 from django_stubs_ext.db.models import TypedModelMeta

-from archivebox.config import CONSTANTS
+from archivebox import DATA_DIR
+from archivebox.misc.util import to_json
+from archivebox.misc.hashing import get_dir_info


-class PagedQuerySet(models.QuerySet):
-    def paged_iterator(self, chunk_size: int = 500):
-        """
-        Iterate rows using bounded keyset pages instead of one streaming cursor.
-
-        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
-        cursor open until the full queryset is exhausted. That is fine for
-        read-only exports, but update/migration code does filesystem work and
-        writes while iterating; a long-lived read cursor there can stretch lock
-        waits across thousands of rows. This respects the queryset's existing
-        filters, order_by(), select_related(), and prefetch_related() state; if
-        no ordering is defined, it falls back to primary-key order.
-        """
-        pk_field = self.model._meta.pk.name
-        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
-
-        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        ordering = []
-        for term in raw_ordering:
-            descending = term.startswith("-")
-            field_name = term[1:] if descending else term
-            if field_name == "pk":
-                field_name = pk_field
-            ordering.append(f"-{field_name}" if descending else field_name)
-
-        ordered_field_names = [term.removeprefix("-") for term in ordering]
-        try:
-            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
-                offset = 0
-                while True:
-                    batch = list(self[offset : offset + chunk_size])
-                    if not batch:
-                        break
-                    yield from batch
-                    offset += chunk_size
-                return
-        except (AttributeError, FieldDoesNotExist):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
-        if not any(field_name in unique_field_names for field_name in ordered_field_names):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        last_values = None
-        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
-        while True:
-            batch_qs = self.order_by(*ordering)
-            if last_values is not None:
-                page_filter = models.Q()
-                for idx, term in enumerate(ordering):
-                    descending = term.startswith("-")
-                    field_name = term[1:] if descending else term
-                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
-                    comparison = "lt" if descending else "gt"
-                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
-                # Anchor the OR tie-breaks with an explicit leading range so
-                # SQLite/PG can seek the existing order index instead of
-                # rescanning its prefix for every keyset page.
-                first_comparison = "lte" if ordering[0].startswith("-") else "gte"
-                batch_qs = batch_qs.filter(
-                    page_filter,
-                    **{f"{ordered_field_names[0]}__{first_comparison}": last_values[0]},
-                )
-
-            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
-            if not batch_rows:
-                break
-
-            pk_idx = value_field_names.index(pk_field)
-            row_ids = [row[pk_idx] for row in batch_rows]
-            rows_by_id = {obj.pk: obj for obj in self.filter(pk__in=row_ids).order_by()}
-
-            for row in batch_rows:
-                row_id = row[pk_idx]
-                obj = rows_by_id.get(row_id)
-                if obj is not None:
-                    yield obj
-
-            last_values = batch_rows[-1][: len(ordered_field_names)]
-
-
-def normalize_config_json_values(config: Any) -> Any:
-    if not isinstance(config, dict):
-        return config
-
-    normalized = dict(config)
-    for key, value in list(normalized.items()):
-        if not isinstance(value, str) or len(value) < 2:
-            continue
-        if value[:1] != '"' or value[-1:] != '"':
-            continue
-        try:
-            decoded = json.loads(value)
-        except ValueError:
-            continue
-        if isinstance(decoded, str):
-            normalized[key] = decoded
-    return normalized
-
-
-def get_or_create_system_user_pk(username="system"):
+def get_or_create_system_user_pk(username='system'):
    User = get_user_model()
    # If there's exactly one superuser, use that for all system operations
    if User.objects.filter(is_superuser=True).count() == 1:
-        return User.objects.filter(is_superuser=True).values_list("pk", flat=True)[0]
+        return User.objects.filter(is_superuser=True).values_list('pk', flat=True)[0]
    # Otherwise get or create the system user
    user, _ = User.objects.get_or_create(
        username=username,
-        defaults={"is_staff": True, "is_superuser": True, "email": "", "password": "!"},
+        defaults={'is_staff': True, 'is_superuser': True, 'email': '', 'password': '!'}
    )
    return user.pk


 class AutoDateTimeField(models.DateTimeField):
    """DateTimeField that automatically updates on save (legacy compatibility)."""
-
    def pre_save(self, model_instance, add):
-        if add or self.attname not in model_instance.__dict__ or not model_instance.__dict__[self.attname]:
+        if add or not getattr(model_instance, self.attname):
            value = timezone.now()
            setattr(model_instance, self.attname, value)
            return value
@@ -172,22 +49,16 @@ class AutoDateTimeField(models.DateTimeField):


 class ModelWithUUID(models.Model):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
    created_at = models.DateTimeField(default=timezone.now, db_index=True)
    modified_at = models.DateTimeField(auto_now=True)
-    created_by = models.ForeignKey(
-        settings.AUTH_USER_MODEL,
-        on_delete=models.CASCADE,
-        default=get_or_create_system_user_pk,
-        null=False,
-        db_index=True,
-    )
+    created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False, db_index=True)

    class Meta(TypedModelMeta):
        abstract = True

-    def __str__(self) -> str:
-        return f"[{self.id}] {self.__class__.__name__}"
+    def __str__(self):
+        return f'[{self.id}] {self.__class__.__name__}'

    @property
    def admin_change_url(self) -> str:
@@ -195,35 +66,46 @@ class ModelWithUUID(models.Model):

    @property
    def api_url(self) -> str:
-        return str(reverse_lazy("api-1:get_any", args=[self.id]))
+        return reverse_lazy('api-1:get_any', args=[self.id])

    @property
    def api_docs_url(self) -> str:
-        return f"/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}"
+        return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}'
+
+    def as_json(self, keys: Iterable[str] = ()) -> dict:
+        default_keys = ('id', 'created_at', 'modified_at')
+        return {key: getattr(self, key) for key in (keys or default_keys) if hasattr(self, key)}
+
+
+class ModelWithSerializers(ModelWithUUID):
+    class Meta(TypedModelMeta):
+        abstract = True
+
+    def as_csv_row(self, keys: Iterable[str] = (), separator: str = ',') -> str:
+        buffer = io.StringIO()
+        csv.writer(buffer, delimiter=separator).writerow(str(getattr(self, key, '')) for key in (keys or self.as_json().keys()))
+        return buffer.getvalue()
+
+    def as_jsonl_row(self, keys: Iterable[str] = (), **json_kwargs) -> str:
+        return json.dumps({key: getattr(self, key, '') for key in (keys or self.as_json().keys())}, sort_keys=True, indent=None, **json_kwargs)


 class ModelWithNotes(models.Model):
    """Mixin for models with a notes field."""
+    notes = models.TextField(blank=True, null=False, default='')

-    notes = models.TextField(blank=True, null=False, default="")
-
-    class Meta(TypedModelMeta):
+    class Meta:
        abstract = True


 class ModelWithHealthStats(models.Model):
    """Mixin for models with health tracking fields."""
-
    num_uses_failed = models.PositiveIntegerField(default=0)
    num_uses_succeeded = models.PositiveIntegerField(default=0)

-    class Meta(TypedModelMeta):
+    class Meta:
        abstract = True

-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
    @property
    def health(self) -> int:
        total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
@@ -231,130 +113,30 @@ class ModelWithHealthStats(models.Model):

    def increment_health_stats(self, success: bool):
        """Atomically increment success or failure counter using F() expression."""
-        field = "num_uses_succeeded" if success else "num_uses_failed"
-        type(self).objects.filter(pk=self.pk).update(
-            **{
-                field: F(field) + 1,
-                "modified_at": timezone.now(),
-            },
-        )
+        field = 'num_uses_succeeded' if success else 'num_uses_failed'
+        type(self).objects.filter(pk=self.pk).update(**{field: F(field) + 1})


 class ModelWithConfig(models.Model):
    """Mixin for models with a JSON config field."""
-
    config = models.JSONField(default=dict, null=True, blank=True, editable=True)

-    class Meta(TypedModelMeta):
+    class Meta:
+        abstract = True
+
+
+class ModelWithOutputDir(ModelWithSerializers):
+    class Meta:
        abstract = True

    def save(self, *args, **kwargs):
-        normalized_config = normalize_config_json_values(self.config)
-        if normalized_config != self.config:
-            self.config = normalized_config
-            update_fields = kwargs.get("update_fields")
-            if update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
        super().save(*args, **kwargs)
-
-
-class ModelWithDeleteAfter(models.Model):
-    delete_after_final_statuses: tuple[str, ...] = ()
-    delete_at = models.DateTimeField(default=None, null=True, blank=True, db_index=True)
-
-    class Meta(TypedModelMeta):
-        abstract = True
-
-    def save(self, *args, **kwargs):
-        update_fields = kwargs.get("update_fields")
-        if self.delete_at is None:
-            self.set_delete_at_from_config()
-            if self.delete_at is not None and update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
-        super().save(*args, **kwargs)
-
-    def get_delete_after_config_value(self):
-        from archivebox.config.common import get_config
-
-        return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
-
-    def set_delete_at_from_config(self, config_value=None) -> bool:
-        if self.delete_at is not None:
-            return False
-
-        from archivebox.config.common import parse_delete_after
-
-        duration = parse_delete_after(self.get_delete_after_config_value() if config_value is None else config_value)
-        if duration is None:
-            return False
-
-        self.delete_at = (self.created_at or timezone.now()) + duration
-        return True
-
-    @classmethod
-    def missing_delete_at_candidates(cls):
-        return cls.objects.none()
-
-    @classmethod
-    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True, backfill_cursors: dict | None = None) -> int:
-        if backfill_missing:
-            # Bound the scanned rows BEFORE evaluating JSON policy/parent
-            # joins. LIMIT on the policy query only bounded its output: finding
-            # no matches could scan the entire history with a SQLite read lock.
-            # The runner keeps a keyset cursor between idle passes, so disabled
-            # policies cannot starve older rows. Restarting safely repeats work.
-            page = cls.objects.order_by("-pk")
-            before = backfill_cursors.get(cls) if backfill_cursors is not None else None
-            if before is not None:
-                page = page.filter(pk__lt=before)
-            page_ids = list(page.values_list("pk", flat=True)[:batch_size])
-            if backfill_cursors is not None:
-                backfill_cursors[cls] = page_ids[-1] if len(page_ids) == batch_size else None
-            missing_delete_at = list(cls.missing_delete_at_candidates().filter(pk__in=page_ids).order_by())
-            for obj in missing_delete_at:
-                if obj.set_delete_at_from_config():
-                    cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
-                        delete_at=obj.delete_at,
-                        modified_at=timezone.now(),
-                    )
-
-        # Keep the expiration sweep anchored on delete_at. Some large tables
-        # have millions of final-status rows but almost no retained rows; adding
-        # the status filter before SQLite has narrowed by delete_at can make the
-        # runner scan the hot status index before it claims new work.
-        due_pks = list(
-            cls.objects.filter(delete_at__isnull=False, delete_at__lte=timezone.now())
-            .order_by("delete_at", "pk")
-            .values_list("pk", flat=True)[:batch_size],
-        )
-        if not due_pks:
-            return 0
-
-        queryset = cls.objects.filter(pk__in=due_pks)
-        if cls.delete_after_final_statuses:
-            queryset = queryset.filter(status__in=cls.delete_after_final_statuses)
-
-        count = 0
-        expired = list(queryset.order_by("delete_at", "pk"))
-        for obj in expired:
-            obj.delete()
-            count += 1
-        return count
-
-
-class ModelWithOutputDir(ModelWithUUID):
-    class Meta(ModelWithUUID.Meta):
-        abstract = True
-
-    _delete_signal_registered = False
-
-    # Saving metadata must not resolve or create output paths. The runner and
-    # explicit artifact writers create their own directories when writing files;
-    # on_commit would still make HTTP requests wait on remote archive storage.
+        self.OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+        self.save_json_index()

    @property
    def output_dir_parent(self) -> str:
-        return f"{self._meta.model_name}s"
+        return f'{self._meta.model_name}s'

    @property
    def output_dir_name(self) -> str:
@@ -362,69 +144,11 @@ class ModelWithOutputDir(ModelWithUUID):

    @property
    def output_dir_str(self) -> str:
-        return f"{self.output_dir_parent}/{self.output_dir_name}"
+        return f'{self.output_dir_parent}/{self.output_dir_name}'

    @property
-    def output_dir(self) -> Path:
-        raise NotImplementedError(f"{self.__class__.__name__} must implement output_dir property")
+    def OUTPUT_DIR(self) -> Path:
+        return DATA_DIR / self.output_dir_str

-    def output_paths_for_delete(self) -> tuple[Path, ...]:
-        return (Path(self.output_dir),)
-
-    @classmethod
-    def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
-        data_dir = CONSTANTS.DATA_DIR.resolve()
-        safe_paths = []
-        for raw_path in paths:
-            path = Path(raw_path)
-            is_safe = False
-            for candidate in (path.absolute(), path.resolve()):
-                try:
-                    candidate.relative_to(data_dir)
-                    is_safe = True
-                    break
-                except ValueError:
-                    continue
-            if not is_safe:
-                raise ValueError(f"Refusing to delete output path outside DATA_DIR: {path}")
-            safe_paths.append(path)
-        return tuple(safe_paths)
-
-    @classmethod
-    def delete_output_paths(cls, paths) -> None:
-        for path in cls.validate_output_paths_for_delete(paths):
-            if path.is_symlink() or path.is_file():
-                path.unlink(missing_ok=True)
-            elif path.is_dir():
-                shutil.rmtree(path)
-            # lstat also catches dangling symlinks and propagates mount and
-            # permission errors rather than mistaking them for absence.
-            try:
-                path.lstat()
-            except FileNotFoundError:
-                continue
-            raise OSError(f"Output path still exists after deletion: {path}")
-
-    def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
-        """Capture output paths before DB deletion and remove them after commit."""
-        paths = self.validate_output_paths_for_delete(self.output_paths_for_delete())
-        transaction.on_commit(lambda: self.delete_output_paths(paths), using=using)
-
-    @classmethod
-    def register_delete_signal(cls) -> None:
-        if cls._delete_signal_registered:
-            return
-
-        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
-            if getattr(origin, "_output_files_deleted", False):
-                return
-            if not isinstance(instance, ModelWithOutputDir):
-                return
-            instance.schedule_delete_cleanup(using=using)
-
-        pre_delete.connect(
-            schedule_output_dir_cleanup,
-            dispatch_uid="archivebox.output_dir_cleanup_on_delete",
-            weak=False,
-        )
-        cls._delete_signal_registered = True
+    def save_json_index(self):
+        (self.OUTPUT_DIR / 'index.json').write_text(to_json(self.as_json()))
@@ -1,232 +1,144 @@
-__package__ = "archivebox.cli"
-__command__ = "archivebox"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox'
 import os
 import sys
-from contextlib import redirect_stdout
 from importlib import import_module

 import rich_click as click
-from rich.console import Console
+from rich import print

 from archivebox.config.version import VERSION

-STDERR = Console(stderr=True)


-if "--debug" in sys.argv:
-    os.environ["DEBUG"] = "True"
-    sys.argv.remove("--debug")
-
-# Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
-# `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
-# the subcommand executes. `--quick-init` is the old server spelling. Strip either from
-# argv here so each subcommand's own click parser never sees it. Ignored for `help` and
-# `init` themselves.
-if "--init" in sys.argv or "--quick-init" in sys.argv:
-    sys.argv = [arg for arg in sys.argv if arg not in ("--init", "--quick-init")]
-    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+if '--debug' in sys.argv:
+    os.environ['DEBUG'] = 'True'
+    sys.argv.remove('--debug')


 class ArchiveBoxGroup(click.Group):
    """lazy loading click group for archivebox commands"""
-
    meta_commands = {
-        "help": "archivebox.cli.archivebox_help.main",
-        "version": "archivebox.cli.archivebox_version.main",
-        "mcp": "archivebox.cli.archivebox_mcp.main",
-        "oneshot": "archivebox.cli.archivebox_oneshot.main",
+        'help': 'archivebox.cli.archivebox_help.main',
+        'version': 'archivebox.cli.archivebox_version.main',
+        'mcp': 'archivebox.cli.archivebox_mcp.main',
    }
    setup_commands = {
-        "init": "archivebox.cli.archivebox_init.main",
-        "install": "archivebox.cli.archivebox_install.main",
-    }
-    # Model commands (CRUD operations via subcommands)
-    model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot.main",
-        "archiveresult": "archivebox.cli.archivebox_archiveresult.main",
-        "tag": "archivebox.cli.archivebox_tag.main",
-        "binary": "archivebox.cli.archivebox_binary.main",
-        "process": "archivebox.cli.archivebox_process.main",
-        "machine": "archivebox.cli.archivebox_machine.main",
-        "persona": "archivebox.cli.archivebox_persona.main",
+        'init': 'archivebox.cli.archivebox_init.main',
+        'install': 'archivebox.cli.archivebox_install.main',
    }
    archive_commands = {
-        # High-level commands
-        "add": "archivebox.cli.archivebox_add.main",
-        "extract": "archivebox.cli.archivebox_extract.main",
-        "list": "archivebox.cli.archivebox_list.main",
-        "remove": "archivebox.cli.archivebox_remove.main",
-        "run": "archivebox.cli.archivebox_run.main",
-        "update": "archivebox.cli.archivebox_update.main",
-        "status": "archivebox.cli.archivebox_status.main",
-        "search": "archivebox.cli.archivebox_search.main",
-        "config": "archivebox.cli.archivebox_config.main",
-        "schedule": "archivebox.cli.archivebox_schedule.main",
-        "server": "archivebox.cli.archivebox_server.main",
-        "shell": "archivebox.cli.archivebox_shell.main",
-        "manage": "archivebox.cli.archivebox_manage.main",
-        # Introspection commands
-        "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
+        'add': 'archivebox.cli.archivebox_add.main',
+        'remove': 'archivebox.cli.archivebox_remove.main',
+        'update': 'archivebox.cli.archivebox_update.main',
+        'search': 'archivebox.cli.archivebox_search.main',
+        'status': 'archivebox.cli.archivebox_status.main',
+        'config': 'archivebox.cli.archivebox_config.main',
+        'schedule': 'archivebox.cli.archivebox_schedule.main',
+        'server': 'archivebox.cli.archivebox_server.main',
+        'shell': 'archivebox.cli.archivebox_shell.main',
+        'manage': 'archivebox.cli.archivebox_manage.main',
+        # Worker/orchestrator commands
+        'orchestrator': 'archivebox.cli.archivebox_orchestrator.main',
+        'worker': 'archivebox.cli.archivebox_worker.main',
+        # Task commands (called by workers as subprocesses)
+        'crawl': 'archivebox.cli.archivebox_crawl.main',
+        'snapshot': 'archivebox.cli.archivebox_snapshot.main',
+        'extract': 'archivebox.cli.archivebox_extract.main',
    }
    all_subcommands = {
        **meta_commands,
        **setup_commands,
-        **model_commands,
        **archive_commands,
    }
    renamed_commands = {
-        "setup": "install",
-        "import": "add",
-        "archive": "add",
+        'setup': 'install',
+        'list': 'search',
+        'import': 'add',
+        'archive': 'add',
+        'export': 'search',
    }
-
+    
    @classmethod
    def get_canonical_name(cls, cmd_name):
        return cls.renamed_commands.get(cmd_name, cmd_name)
-
-    @classmethod
-    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
-        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
-        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)
-
-    @classmethod
-    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
-        if not cls._needs_django_for_lazy_import(cmd_name):
-            return
-
-        from django.apps import apps
-
-        if apps.ready:
-            return
-
-        from archivebox.config.django import setup_django
-
-        setup_django()
+    

    def get_command(self, ctx, cmd_name):
        # handle renamed commands
        if cmd_name in self.renamed_commands:
            new_name = self.renamed_commands[cmd_name]
-            STDERR.print(
-                f" [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`",
-            )
+            print(f' [violet]Hint:[/violet] `archivebox {cmd_name}` has been renamed to `archivebox {new_name}`')
            cmd_name = new_name
            ctx.invoked_subcommand = cmd_name
-
+        
        # handle lazy loading of commands
        if cmd_name in self.all_subcommands:
-            self._setup_django_for_lazy_import(cmd_name)
            return self._lazy_load(cmd_name)
-
+        
        # fall-back to using click's default command lookup
        return super().get_command(ctx, cmd_name)

    @classmethod
-    def _lazy_load(cls, cmd_name_or_path):
-        import_path = cls.all_subcommands.get(cmd_name_or_path)
-        if import_path is None:
-            import_path = cmd_name_or_path
-        modname, funcname = import_path.rsplit(".", 1)
-
+    def _lazy_load(cls, cmd_name):
+        import_path = cls.all_subcommands[cmd_name]
+        modname, funcname = import_path.rsplit('.', 1)
+        
+        # print(f'LAZY LOADING {import_path}')
        mod = import_module(modname)
-        func = vars(mod)[funcname]
-
-        if func.__doc__ is None:
-            raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
-
+        func = getattr(mod, funcname)
+        
+        if not hasattr(func, '__doc__'):
+            raise ValueError(f'lazy loading of {import_path} failed - no docstring found on method')
+        
+        # if not isinstance(cmd, click.BaseCommand):
+            # raise ValueError(f'lazy loading of {import_path} failed - not a click command')
+            
        return func


@click.group(cls=ArchiveBoxGroup, invoke_without_command=True)
-@click.option("--help", "-h", is_flag=True, help="Show help")
-@click.version_option(VERSION, "-v", "--version", package_name="archivebox", message="%(version)s")
+@click.option('--help', '-h', is_flag=True, help='Show help')
+@click.version_option(VERSION, '-v', '--version', package_name='archivebox', message='%(version)s')
@click.pass_context
 def cli(ctx, help=False):
    """ArchiveBox: The self-hosted internet archive"""
-
+    
    subcommand = ArchiveBoxGroup.get_canonical_name(ctx.invoked_subcommand)
-
+    
    # if --help is passed or no subcommand is given, show custom help message
    if help or ctx.invoked_subcommand is None:
-        ctx.invoke(ctx.command.get_command(ctx, "help"))
-
-    # if the subcommand is in archive_commands or model_commands,
+        ctx.invoke(ctx.command.get_command(ctx, 'help'))
+    
+    # if the subcommand is in the archive_commands dict and is not 'manage',
    # then we need to set up the django environment and check that we're in a valid data folder
-    wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
-    if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
+    if subcommand in ArchiveBoxGroup.archive_commands:
+        # print('SETUP DJANGO AND CHECK DATA FOLDER')
        try:
-            # RUNSERVER_WORKER sets the web-worker markers on its own child.
-            # Setting them on the server parent leaks them through supervisord
-            # into runners waiting for takeover, which then register themselves
-            # as web workers and trigger spurious runner_watch reloads.
            from archivebox.config.django import setup_django
-            from archivebox.misc.checks import check_data_folder, check_migrations
-
+            from archivebox.misc.checks import check_data_folder
            setup_django()
-            if os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1" and subcommand not in ("init", "help"):
-                # Universal `--init` was passed: build/upgrade the data folder before
-                # the regular preflight runs, so it succeeds on a fresh dir and an
-                # out-of-date schema both. Drop the env var afterwards so spawned
-                # subprocesses (supervisord workers, daphne, runner, etc.) inherit
-                # a clean env and don't re-trigger init in every child.
-                from archivebox.cli.archivebox_init import init as archivebox_init
-
-                archivebox_init(quick=True)
-                os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
            check_data_folder()
-            if subcommand != "update" and not check_migrations(auto_apply=True):
-                from archivebox.machine.models import Machine
-
-                # Host registration and config reconciliation belong to CLI
-                # startup; get_config() is also used by read-only HTTP views.
-                Machine.current()
        except Exception as e:
-            STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
-            if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
+            print(f'[red][X] Error setting up Django or checking data folder: {e}[/red]', file=sys.stderr)
+            if subcommand not in ('manage', 'shell'):   # not all management commands need django to be setup beforehand
                raise
+            

-
-def main(args=None, prog_name=None):
-    from archivebox.config.constants import CONSTANTS
-
-    os.environ.setdefault("ABX_PLUGINS_DIR", str(CONSTANTS.USER_PLUGINS_DIR))
-
+def main(args=None, prog_name=None, stdin=None):
    # show `docker run archivebox xyz` in help messages if running in docker
-    IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
+    IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'TRUE', 'yes')
    IS_TTY = sys.stdin.isatty()
-    prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
+    prog_name = prog_name or (f'docker compose run{"" if IS_TTY else " -T"} archivebox' if IN_DOCKER else 'archivebox')
+    
+    # stdin param allows passing input data from caller (used by __main__.py)
+    # currently not used by click-based CLI, but kept for backwards compatibility

-    previous_unraisablehook = sys.unraisablehook
-
-    def ignore_shutdown_unraisable(unraisable):
-        if isinstance(unraisable.exc_value, (KeyboardInterrupt, SystemExit)):
-            return
-        previous_unraisablehook(unraisable)
-
-    sys.unraisablehook = ignore_shutdown_unraisable
    try:
-        cli(args=args, prog_name=prog_name, standalone_mode=False)
-    except click.Abort:
-        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
-        raise SystemExit(130) from None
-    except click.ClickException as err:
-        err.show()
-        raise SystemExit(err.exit_code) from None
-    except click.exceptions.Exit as err:
-        raise SystemExit(err.exit_code) from None
+        cli(args=args, prog_name=prog_name)
    except KeyboardInterrupt:
-        STDERR.print("\n[red][X] Got CTRL+C. Exiting...[/red]")
-        raise SystemExit(130) from None
-    finally:
-        sys.unraisablehook = previous_unraisablehook
-        supervisor_util = sys.modules.get("archivebox.workers.supervisord_util")
-        if supervisor_util is not None:
-            # Short commands can start a foreground-owned search daemon too.
-            # Finish its cleanup before another CLI can borrow a dying stack.
-            with redirect_stdout(sys.stderr):
-                supervisor_util.stop_own_supervisord_process()
+        print('\n\n[red][X] Got CTRL+C. Exiting...[/red]')


-if __name__ == "__main__":
+if __name__ == '__main__':
    main()
@@ -1,446 +1,160 @@
 #!/usr/bin/env python3
-from __future__ import annotations

-__package__ = "archivebox.cli"
-__command__ = "archivebox add"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox add'

 import sys
-import os
-import signal
-from pathlib import Path

-from typing import Any, TYPE_CHECKING
+from typing import TYPE_CHECKING

 import rich_click as click

-from archivebox.config import CONSTANTS
+from django.utils import timezone
+from django.db.models import QuerySet
+
 from archivebox.misc.util import enforce_types, docstring
-from archivebox.misc.util import parse_filesize_to_bytes
+from archivebox import CONSTANTS
+from archivebox.config.common import ARCHIVING_CONFIG
+from archivebox.config.permissions import USER, HOSTNAME


 if TYPE_CHECKING:
-    from django.db.models import QuerySet
-    from archivebox.crawls.models import Crawl
    from archivebox.core.models import Snapshot


-def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
-    from archivebox.misc.jsonl import read_args_or_stdin
-    from archivebox.misc.util import validate_url
-
-    urls: list[str] = []
-    for record in read_args_or_stdin(args):
-        url = record.get("url")
-        if isinstance(url, str) and url:
-            try:
-                urls.append(validate_url(url))
-            except ValueError as err:
-                raise click.BadParameter(str(err), param_hint="URL") from err
-
-        urls_field = record.get("urls")
-        if isinstance(urls_field, str):
-            for line in urls_field.splitlines():
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    try:
-                        urls.append(validate_url(line))
-                    except ValueError as err:
-                        raise click.BadParameter(str(err), param_hint="URL") from err
-
-    return urls
-
-
@enforce_types
-def add(
-    urls: str | list[str],
-    snapshot_ids: list[str] | None = None,
-    depth: int | str = 0,
-    max_urls: int = 0,
-    crawl_max_size: int | str = 0,
-    crawl_timeout: int = 0,
-    snapshot_max_size: int | str = 0,
-    crawl_max_concurrent_snapshots: int | None = None,
-    tag: str = "",
-    url_allowlist: str = "",
-    url_denylist: str = "",
-    parser: str = "auto",
-    plugins: str = "",
-    persona: str = "Default",
-    index_only: bool = False,
-    bg: bool = False,
-    created_by_id: int | None = None,
-    config: dict[str, Any] | None = None,
-) -> tuple[Crawl, QuerySet[Snapshot]]:
-    """Add a URL list or imported URL document to a new Crawl."""
+def add(urls: str | list[str],
+        depth: int | str=0,
+        tag: str='',
+        parser: str="auto",
+        plugins: str="",
+        persona: str='Default',
+        overwrite: bool=False,
+        update: bool=not ARCHIVING_CONFIG.ONLY_NEW,
+        index_only: bool=False,
+        bg: bool=False,
+        created_by_id: int | None=None) -> QuerySet['Snapshot']:
+    """Add a new URL or list of URLs to your archive.
+
+    The flow is:
+    1. Save URLs to sources file
+    2. Create Crawl with URLs and max_depth
+    3. Orchestrator creates Snapshots from Crawl URLs (depth=0)
+    4. Orchestrator runs parser extractors on root snapshots
+    5. Parser extractors output to urls.jsonl
+    6. URLs are added to Crawl.urls and child Snapshots are created
+    7. Repeat until max_depth is reached
+    """

    from rich import print

    depth = int(depth)
-    max_urls = int(max_urls or 0)
-    crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
-    crawl_timeout = int(crawl_timeout or 0)
-    snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    from archivebox.config.permissions import USER, HOSTNAME
-    from archivebox.config.common import get_config

-    config_overrides = dict(config or {})
-    runtime_config = get_config()
-    crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
-    if crawl_max_concurrent_snapshots is None:
-        crawl_max_concurrent_snapshots = runtime_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
-    crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
-
-    if depth not in (0, 1, 2, 3, 4):
-        raise ValueError("Depth must be 0-4")
-    if max_urls < 0:
-        raise ValueError("max_urls must be >= 0")
-    if crawl_max_size < 0:
-        raise ValueError("crawl_max_size must be >= 0")
-    if crawl_timeout < 0:
-        raise ValueError("crawl_timeout must be >= 0")
-    if snapshot_max_size < 0:
-        raise ValueError("snapshot_max_size must be >= 0")
-    if crawl_max_concurrent_snapshots < 1:
-        raise ValueError("crawl_max_concurrent_snapshots must be >= 1")
+    assert depth in (0, 1, 2, 3, 4), 'Depth must be 0-4'

    # import models once django is set up
+    from archivebox.core.models import Snapshot
    from archivebox.crawls.models import Crawl
    from archivebox.base_models.models import get_or_create_system_user_pk
-    from archivebox.personas.models import Persona
-    from archivebox.misc.logging_util import printable_filesize
-    from archivebox.misc.system import get_dir_size
-    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-    from django.utils import timezone
+    from archivebox.workers.orchestrator import Orchestrator

    created_by_id = created_by_id or get_or_create_system_user_pk()
-    started_at = timezone.now()

-    if isinstance(urls, str):
-        source_text = urls
-    else:
-        from archivebox.misc.util import validate_url
-
-        source_text = "\n".join(validate_url(str(url)) for url in urls)
+    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
+    sources_file = CONSTANTS.SOURCES_DIR / f'{timezone.now().strftime("%Y-%m-%d__%H-%M-%S")}__cli_add.txt'
+    sources_file.parent.mkdir(parents=True, exist_ok=True)
+    sources_file.write_text(urls if isinstance(urls, str) else '\n'.join(urls))

    # 2. Create a new Crawl with inline URLs
-    # Foreground add must claim runner ownership before publishing runnable
-    # work. Otherwise an existing server runner can lease the new Crawl in the
-    # gap between Crawl.objects.create() and current_command(), then be killed
-    # by the newer add process halfway through its first Snapshot hook.
-    command = None
-    if not bg and not index_only:
-        from archivebox.machine.models import Process
-        from archivebox.core.takeover_util import current_command
-
-        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR)
-
    cli_args = [*sys.argv]
-    if cli_args[0].lower().endswith("archivebox"):
-        cli_args[0] = "archivebox"
-    cmd_str = " ".join(cli_args)
+    if cli_args[0].lower().endswith('archivebox'):
+        cli_args[0] = 'archivebox'
+    cmd_str = ' '.join(cli_args)

    timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")

-    persona_name = (persona or "Default").strip() or "Default"
-    plugins = plugins or ""
-    persona_obj = Persona.get_or_create_named(persona_name)
-    effective_persona_config = get_config(persona=persona_obj)
+    # Read URLs directly into crawl
+    urls_content = sources_file.read_text()

-    crawl_config = {
-        "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
-        **({"INDEX_ONLY": True} if index_only else {}),
-        **({"PLUGINS": plugins} if plugins else {}),
-        **(
-            {"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots}
-            if crawl_max_concurrent_snapshots_override
-            and crawl_max_concurrent_snapshots != int(effective_persona_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS)
-            else {}
-        ),
-        **({"CRAWL_MAX_URLS": max_urls} if max_urls else {}),
-        **({"CRAWL_MAX_SIZE": crawl_max_size} if crawl_max_size else {}),
-        **({"CRAWL_TIMEOUT": crawl_timeout} if crawl_timeout else {}),
-        **({"SNAPSHOT_MAX_SIZE": snapshot_max_size} if snapshot_max_size else {}),
-        **({"PARSER": parser} if parser != "auto" else {}),
-        **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
-        **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
-    }
-    # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
-    # priority for crawl-frozen keys. Runtime-derived execution keys are
-    # stripped by Crawl.save() and rederived when hooks run.
-    crawl_config.update(config_overrides)
+    crawl = Crawl.objects.create(
+        urls=urls_content,
+        max_depth=depth,
+        tags_str=tag,
+        label=f'{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]',
+        created_by_id=created_by_id,
+        config={
+            'ONLY_NEW': not update,
+            'INDEX_ONLY': index_only,
+            'OVERWRITE': overwrite,
+            'PLUGINS': plugins,
+            'DEFAULT_PERSONA': persona or 'Default',
+            'PARSER': parser,
+        }
+    )

-    # Crawl.urls is the user's submitted source, not a derived work queue for
-    # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
-    # callers audit or resume imports without losing RSS/Netscape/JSON metadata
-    # that is not representable as one plain URL per line.
-    try:
-        crawl = Crawl.objects.create(
-            urls=source_text,
-            max_depth=depth,
-            tags_str=tag,
-            persona_id=persona_obj.id,
-            label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
-            created_by_id=created_by_id,
-            status=Crawl.StatusChoices.QUEUED,
-            retry_at=None if index_only else timezone.now(),
-            config=crawl_config,
-        )
-        first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
-    except BaseException:
-        if command is not None:
-            command.mark_exited(exit_code=1)
-        raise
-    if command is not None:
-        command.url = first_url
-        command.save(update_fields=["url", "modified_at"])
+    print(f'[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]')
+    first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ''
+    print(f'    [dim]First URL: {first_url}[/dim]')

-    print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
-    print(f"    [dim]First URL: {first_url}[/dim]")
-
-    # The runner parses Crawl.urls after claiming the Crawl, then persists the
-    # resulting depth-0 Snapshot facts. Recursive discoveries start at depth 1.
+    # 3. The CrawlMachine will create the root Snapshot when started
+    #    If URLs are from a file: first URL = file:///path/to/sources/...txt
+    #    Parser extractors will run on it and discover more URLs
+    #    Those URLs become child Snapshots (depth=1)

    if index_only:
-        print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
-        return crawl, crawl.snapshot_set.none()
+        # Just create the crawl but don't start processing
+        print('[yellow]\\[*] Index-only mode - crawl created but not started[/yellow]')
+        # Create snapshots for all URLs in the crawl
+        for url in crawl.get_urls_list():
+            Snapshot.objects.update_or_create(
+                crawl=crawl, url=url,
+                defaults={
+                    'status': Snapshot.INITIAL_STATE,
+                    'retry_at': timezone.now(),
+                    'timestamp': str(timezone.now().timestamp()),
+                    'depth': 0,
+                },
+            )
+        return crawl.snapshot_set.all()

-    # 5. Start the crawl runner to process the queue
-    #    The runner will:
-    #    - Process Crawl -> create Snapshots from all URLs
-    #    - Process Snapshots -> run extractors
-    #    - Parser extractors discover new URLs -> create child Snapshots
+    # 5. Start the orchestrator to process the queue
+    #    The orchestrator will:
+    #    - Process Crawl -> create root Snapshot
+    #    - Process root Snapshot -> run parser extractors -> discover URLs
+    #    - Create child Snapshots from discovered URLs
+    #    - Process child Snapshots -> run extractors
    #    - Repeat until max_depth reached

    if bg:
-        # Background mode: just queue work and return (background runner via server will pick it up).
-        print(
-            "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
-        )
+        # Background mode: just queue work and return (orchestrator via server will pick it up)
+        print('[yellow]\\[*] URLs queued. Orchestrator will process them (run `archivebox server` if not already running).[/yellow]')
    else:
-        # Foreground mode: run full crawl runner until all work is done
-        print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
-        from archivebox.core.takeover_util import command_owns_foreground_runner, standby_until_foreground_runner_needed
-        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
-
-        assert command is not None
-        exit_code = 0
-        interactive_interrupts = sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
-
-        def interrupt_hook() -> None:
-            raise KeyboardInterrupt
-
-        def crawl_is_complete() -> bool:
-            crawl.refresh_from_db(fields=["status"])
-            return crawl.status not in crawl.RUNNABLE_STATES
-
-        try:
-            try:
-                with (
-                    foreground_shutdown_signals(
-                        first_signal_message=None,
-                        interrupt_handlers={signal.SIGINT: interrupt_hook},
-                    ),
-                    foreground_parent_watchdog(),
-                ):
-                    while True:
-                        standby = standby_until_foreground_runner_needed(
-                            command,
-                            data_dir=CONSTANTS.DATA_DIR,
-                            work_is_complete=crawl_is_complete,
-                        )
-                        if standby["work_completed"]:
-                            break
-                        exit_code = run_runner_worker(
-                            ["--crawl-id", str(crawl.id)],
-                            name=f"worker_runner_add_{os.getpid()}",
-                            interactive_interrupts=interactive_interrupts,
-                            config=get_config(crawl=crawl),
-                        )
-                        crawl.refresh_from_db(fields=["status", "retry_at"])
-                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
-                            break
-                        # A shared supervisord can be stopped by another
-                        # foreground owner (for example `archivebox server`
-                        # shutting down) and stop the one-shot add runner while
-                        # this add's crawl is still runnable. Keep the add
-                        # process alive so it can re-enter the normal
-                        # foreground-runner ownership loop and finish its crawl.
-                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
-                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
-                            continue
-                        if exit_code == 0:
-                            break
-                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
-                            continue
-                        raise SystemExit(exit_code)
-            except KeyboardInterrupt:
-                exit_code = 130
-                print("\n[red][X] archivebox add interrupted.[/red]")
-                print("[yellow]Hint: resume this crawl with:[/yellow]")
-                print(f"    [green]archivebox run --crawl-id={crawl.id}[/green]")
-                raise SystemExit(exit_code)
-        finally:
-            command.mark_exited(exit_code=exit_code)
-            stop_own_supervisord_process()
-
-        # Print summary for foreground runs
-        try:
-            crawl.refresh_from_db()
-            try:
-                from django.db.models import Count, Sum
-
-                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("output_size"))
-                snapshots_count = int(totals["snapshot_count"] or 0)
-                total_bytes = int(totals["total_bytes"] or 0)
-            except Exception:
-                snapshots_count = crawl.snapshot_set.count()
-                total_bytes, _, _ = get_dir_size(crawl.output_dir)
-            total_size = printable_filesize(total_bytes)
-            total_time = timezone.now() - started_at
-            total_seconds = int(total_time.total_seconds())
-            mins, secs = divmod(total_seconds, 60)
-            hours, mins = divmod(mins, 60)
-            if hours:
-                duration_str = f"{hours}h {mins}m {secs}s"
-            elif mins:
-                duration_str = f"{mins}m {secs}s"
-            else:
-                duration_str = f"{secs}s"
-
-            # Output dir relative to DATA_DIR
-            try:
-                rel_output = Path(crawl.output_dir).relative_to(CONSTANTS.DATA_DIR)
-                rel_output_str = f"./{rel_output}"
-            except Exception:
-                rel_output_str = str(crawl.output_dir)
-
-            from archivebox.core.routes_util import build_admin_url
-
-            admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
-
-            if snapshots_count:
-                print("\n[bold]snapshot output saved to:[/bold]")
-                for snapshot in crawl.snapshot_set.order_by("created_at").iterator():
-                    click.echo(f"  {snapshot.output_dir}")
-            print("\n[bold]crawl logs and setup:[/bold]")
-            click.echo(f"  {rel_output_str}")
-            click.echo(f"  {admin_url}")
-            print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
-            print(f"[bold]total size:[/bold] {total_size}")
-            print(f"[bold]total time:[/bold] {duration_str}")
-        except Exception:
-            # Summary is best-effort; avoid failing the command if something goes wrong
-            pass
+        # Foreground mode: run orchestrator inline until all work is done
+        print(f'[green]\\[*] Starting orchestrator to process crawl...[/green]')
+        orchestrator = Orchestrator(exit_on_idle=True)
+        orchestrator.runloop()  # Block until complete

    # 6. Return the list of Snapshots in this crawl
-    snapshots = crawl.snapshot_set.all()
-    return crawl, snapshots
+    return crawl.snapshot_set.all()


@click.command()
-@click.option(
-    "--depth",
-    "-d",
-    type=click.Choice([str(i) for i in range(5)]),
-    default="0",
-    help="Recursively archive linked pages up to N hops away",
-)
-@click.option("--max-urls", type=int, default=0, help="Maximum number of URLs to snapshot for this crawl (0 = unlimited)")
-@click.option("--crawl-max-size", default="0", help="Maximum total crawl size in bytes or units like 45mb / 1gb (0 = unlimited)")
-@click.option("--crawl-timeout", type=int, default=0, help="Maximum total crawl runtime in seconds (0 = unlimited)")
-@click.option("--snapshot-max-size", default="0", help="Maximum per-snapshot size in bytes or units like 45mb / 1gb (0 = unlimited)")
-@click.option("--crawl-max-concurrent-snapshots", type=int, default=None, help="Maximum snapshots to archive concurrently within one crawl")
-@click.option("--tag", "-t", default="", help="Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3")
-@click.option("--url-allowlist", "--domain-allowlist", default="", help="Comma-separated URL/domain allowlist for this crawl")
-@click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
-@click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
-@click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
-@click.option("--extract", default="", help="Comma-separated plugins or output types to extract e.g. title,pdf,text/html,image,...")
-@click.option("--persona", default="Default", help="Authentication profile to use when archiving")
-@click.option(
-    "--only-new/--no-only-new",
-    "only_new",
-    default=None,
-    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
-    "Pass --no-only-new to force re-archive of URLs that already exist.",
-)
-@click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
-@click.option("--overwrite", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
-@click.option("--update", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
-@click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
-@click.argument("urls", nargs=-1, type=click.Path())
+@click.option('--depth', '-d', type=click.Choice([str(i) for i in range(5)]), default='0', help='Recursively archive linked pages up to N hops away')
+@click.option('--tag', '-t', default='', help='Comma-separated list of tags to add to each snapshot e.g. tag1,tag2,tag3')
+@click.option('--parser', default='auto', help='Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)')
+@click.option('--plugins', '-p', default='', help='Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...')
+@click.option('--persona', default='Default', help='Authentication profile to use when archiving')
+@click.option('--overwrite', '-F', is_flag=True, help='Overwrite existing data if URLs have been archived previously')
+@click.option('--update', is_flag=True, default=ARCHIVING_CONFIG.ONLY_NEW, help='Retry any previously skipped/failed URLs when re-adding them')
+@click.option('--index-only', is_flag=True, help='Just add the URLs to the index without archiving them now')
+@click.option('--bg', is_flag=True, help='Run archiving in background (start orchestrator and return immediately)')
+@click.argument('urls', nargs=-1, type=click.Path())
@docstring(add.__doc__)
 def main(**kwargs):
    """Add a new URL or list of URLs to your archive"""

-    from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-
-    with foreground_shutdown_signals(), foreground_parent_watchdog():
-        raw_urls = kwargs.pop("urls")
-        if raw_urls:
-            urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
-        elif not sys.stdin.isatty():
-            urls = sys.stdin.read()
-        else:
-            urls = []
-        if not urls:
-            raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
-        if int(kwargs.get("max_urls") or 0) < 0:
-            raise click.BadParameter("max_urls must be 0 or a positive integer.", param_hint="--max-urls")
-        if int(kwargs.get("crawl_timeout") or 0) < 0:
-            raise click.BadParameter("crawl_timeout must be 0 or a positive integer.", param_hint="--crawl-timeout")
-        try:
-            kwargs["crawl_max_size"] = parse_filesize_to_bytes(kwargs.get("crawl_max_size"))
-        except ValueError as err:
-            raise click.BadParameter(str(err), param_hint="--crawl-max-size") from err
-        try:
-            kwargs["snapshot_max_size"] = parse_filesize_to_bytes(kwargs.get("snapshot_max_size"))
-        except ValueError as err:
-            raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
-        if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
-            raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
-
-        # Translate --only-new/--no-only-new into a crawl config override.
-        # add() takes config overrides as a dict; no per-flag kwargs.
-        overwrite = kwargs.pop("overwrite", False)
-        update = kwargs.pop("update", False)
-        only_new = kwargs.pop("only_new", None)
-        extract = kwargs.pop("extract", "")
-        if overwrite or update:
-            only_new = False
-        if only_new is not None:
-            kwargs["config"] = {"ONLY_NEW": bool(only_new)}
-        if extract:
-            from archivebox.plugins.discovery import get_plugin_catalog
-
-            catalog = get_plugin_catalog()
-            tokens = [token.strip() for token in extract.split(",") if token.strip()]
-            plugin_names = {name.lower(): name for name in catalog}
-            selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
-            selected += catalog.matching_output(tokens)
-            if not selected:
-                raise click.UsageError(f"No plugins found matching extract types: {extract}")
-            existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
-            kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
-
-        from archivebox.machine.models import Process
-
-        # CLI commands own their process record and logs. Queue model saves
-        # must not register a process implicitly: HTTP add() shares that path.
-        process = Process.current()
-        exit_code = 0
-        try:
-            add(urls=urls, **kwargs)
-        except BaseException:
-            exit_code = 1
-            raise
-        finally:
-            # --bg/--index-only return without the foreground runner's cleanup.
-            # Leaving them RUNNING makes the active-process index accumulate
-            # historical CLI commands forever, defeating bounded recovery.
-            if process.status == Process.StatusChoices.RUNNING:
-                process.mark_exited(exit_code=exit_code)
+    add(**kwargs)


-if __name__ == "__main__":
+if __name__ == '__main__':
    main()
@@ -1,383 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox archiveresult <action> [args...] [--filters]
-
-Manage ArchiveResult records (plugin extraction results).
-
-Actions:
-    create  - Emit plugin extraction request records for Snapshots
-    list    - List ArchiveResults as JSONL (with optional filters)
-    update  - Update ArchiveResults from stdin JSONL
-    delete  - Delete ArchiveResults from stdin JSONL
-
-Examples:
-    # Emit extraction requests; `archivebox run` schedules their parent snapshots
-    archivebox snapshot list --status=queued | archivebox archiveresult create
-    archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
-
-    # List with filters
-    archivebox archiveresult list --status=failed
-    archivebox archiveresult list --plugin=screenshot --status=succeeded
-
-    # Delete
-    archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
-
-    # Re-run failed extractions
-    archivebox archiveresult list --status=failed | archivebox run
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox archiveresult"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_util import apply_filters
-
-
-def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
-    return {
-        "type": "ArchiveResult",
-        "snapshot_id": str(snapshot_id),
-        "plugin": plugin,
-        "hook_name": hook_name,
-        "status": status,
-    }
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_archiveresults(
-    snapshot_id: str | None = None,
-    plugin: str | None = None,
-    status: str = "queued",
-) -> int:
-    """
-    Create ArchiveResult request records for Snapshots.
-
-    Reads Snapshot records from stdin and emits ArchiveResult request JSONL.
-    Pass-through: Non-Snapshot/ArchiveResult records are output unchanged.
-    If --plugin is specified, only emits requests for that plugin.
-    Otherwise, emits requests for all enabled snapshot hooks.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.config.common import get_config
-    from archivebox.plugins.hooks import discover_hooks
-    from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
-    from archivebox.core.models import Snapshot
-
-    is_tty = sys.stdout.isatty()
-
-    # If snapshot_id provided directly, use that
-    if snapshot_id:
-        try:
-            snapshots = [Snapshot.objects.get(id=snapshot_id)]
-            pass_through_records = []
-        except Snapshot.DoesNotExist:
-            rprint(f"[red]Snapshot not found: {snapshot_id}[/red]", file=sys.stderr)
-            return 1
-    else:
-        # Read from stdin
-        records = list(read_stdin())
-        if not records:
-            rprint("[yellow]No Snapshot records provided via stdin[/yellow]", file=sys.stderr)
-            return 1
-
-        # Separate snapshot records from pass-through records
-        snapshot_ids = []
-        pass_through_records = []
-
-        for record in records:
-            record_type = record.get("type", "")
-
-            if record_type == TYPE_SNAPSHOT:
-                # Pass through the Snapshot record itself
-                pass_through_records.append(record)
-                if record.get("id"):
-                    snapshot_ids.append(record["id"])
-
-            elif record_type == TYPE_ARCHIVERESULT:
-                # ArchiveResult records: pass through if they have an id
-                if record.get("id"):
-                    pass_through_records.append(record)
-                # If no id, we could create it, but for now just pass through
-                else:
-                    pass_through_records.append(record)
-
-            elif record_type:
-                # Other typed records (Crawl, Tag, etc): pass through
-                pass_through_records.append(record)
-
-            elif record.get("id"):
-                # Untyped record with id - assume it's a snapshot ID
-                snapshot_ids.append(record["id"])
-
-        # Output pass-through records first
-        if not is_tty:
-            for record in pass_through_records:
-                write_record(record)
-
-        if not snapshot_ids:
-            if pass_through_records:
-                rprint(f"[dim]Passed through {len(pass_through_records)} records, no new snapshots to process[/dim]", file=sys.stderr)
-                return 0
-            rprint("[yellow]No valid Snapshot IDs in input[/yellow]", file=sys.stderr)
-            return 1
-
-        snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids))
-
-    if not snapshots:
-        rprint("[yellow]No matching snapshots found[/yellow]", file=sys.stderr)
-        return 0 if pass_through_records else 1
-
-    created_count = 0
-    for snapshot in snapshots:
-        config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
-        hooks = [
-            hook
-            for hook in discover_hooks("Snapshot", filter_disabled=not plugin, config=config)
-            if not plugin or hook.parent.name == plugin
-        ]
-        for hook_path in hooks:
-            hook_name = hook_path.stem
-            plugin_name = hook_path.parent.name
-            if not is_tty:
-                write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
-            created_count += 1
-
-    rprint(f"[green]Created {created_count} extraction request records[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_archiveresults(
-    status: str | None = None,
-    plugin: str | None = None,
-    snapshot_id: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List ArchiveResults as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import ArchiveResult
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = ArchiveResult.objects.all().order_by("-start_ts")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "plugin": plugin,
-        "snapshot_id": snapshot_id,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for result in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "succeeded": "green",
-                "failed": "red",
-                "skipped": "dim",
-                "noresults": "dim",
-                "backoff": "magenta",
-            }.get(result.status, "dim")
-            rprint(
-                f"[{status_color}]{result.status:10}[/{status_color}] {result.plugin:15} [dim]{result.id}[/dim] {result.snapshot.url[:40]}",
-            )
-        else:
-            write_record(result.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} archive results[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_archiveresults(
-    status: str | None = None,
-) -> int:
-    """
-    Update ArchiveResults from stdin JSONL.
-
-    Reads ArchiveResult records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.core.models import ArchiveResult
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        result_id = record.get("id")
-        if not result_id:
-            continue
-
-        try:
-            result = ArchiveResult.objects.get(id=result_id)
-
-            # Apply updates from CLI flags
-            if status:
-                result.status = status
-
-            result.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(result.to_json())
-
-        except ArchiveResult.DoesNotExist:
-            rprint(f"[yellow]ArchiveResult not found: {result_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} archive results[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_archiveresults(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete ArchiveResults from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.core.models import ArchiveResult
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    result_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not result_ids:
-        rprint("[yellow]No valid archive result IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    results = ArchiveResult.objects.filter(id__in=result_ids)
-    count = results.count()
-
-    if count == 0:
-        rprint("[yellow]No matching archive results found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} archive results (dry run)[/yellow]", file=sys.stderr)
-        for result in results[:10]:
-            rprint(f"  [dim]{result.id}[/dim] {result.plugin} {result.snapshot.url[:40]}", file=sys.stderr)
-        if count > 10:
-            rprint(f"  ... and {count - 10} more", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = results.delete()
-    rprint(f"[green]Deleted {deleted_count} archive results[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage ArchiveResult records (plugin extraction results)."""
-    pass
-
-
-@main.command("create")
-@click.option("--snapshot-id", help="Snapshot ID to create results for")
-@click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
-    """Emit Snapshot plugin extraction requests as JSONL."""
-    sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
-
-
-@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, succeeded, failed, skipped)")
-@click.option("--plugin", "-p", help="Filter by plugin name")
-@click.option("--snapshot-id", help="Filter by snapshot ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(
-    status: str | None,
-    plugin: str | None,
-    snapshot_id: str | None,
-    limit: int | None,
-):
-    """List ArchiveResults as JSONL."""
-    sys.exit(
-        list_archiveresults(
-            status=status,
-            plugin=plugin,
-            snapshot_id=snapshot_id,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--status", "-s", help="Set status")
-def update_cmd(status: str | None):
-    """Update ArchiveResults from stdin JSONL."""
-    sys.exit(update_archiveresults(status=status))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete ArchiveResults from stdin JSONL."""
-    sys.exit(delete_archiveresults(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,319 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-archivebox binary <action> [args...] [--filters]
-
-Manage Binary records (detected executables like chrome, wget, etc.).
-
-Actions:
-    create  - Create/register a Binary
-    list    - List Binaries as JSONL (with optional filters)
-    update  - Update Binaries from stdin JSONL
-    delete  - Delete Binaries from stdin JSONL
-
-Examples:
-    # List all binaries
-    archivebox binary list
-
-    # List specific binary
-    archivebox binary list --name=chrome
-
-    # List binaries with specific version
-    archivebox binary list --version__icontains=120
-
-    # Delete old binary entries
-    archivebox binary list --name=chrome | archivebox binary delete --yes
-"""
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox binary"
-
-import sys
-
-import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_util import apply_filters
-
-
-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_binary(
-    name: str,
-    abspath: str,
-    version: str = "",
-) -> int:
-    """
-    Create/register a Binary.
-
-    Exit codes:
-        0: Success
-        1: Failure
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    if not name or not abspath:
-        rprint("[red]Both --name and --abspath are required[/red]", file=sys.stderr)
-        return 1
-
-    try:
-        from archivebox.machine.models import Machine
-
-        machine = Machine.current()
-        created = not Binary.objects.filter(
-            machine=machine,
-            name=name,
-            abspath=abspath,
-            version=version,
-        ).exists()
-
-        # Mirror the Binary model lifecycle used elsewhere in the system so CLI
-        # records are owned by the current machine and can be safely piped into
-        # `archivebox run` without creating invalid rows missing machine_id.
-        binary = Binary.from_json(
-            {
-                "name": name,
-                "abspath": abspath,
-                "version": version,
-                "binproviders": "env",
-                "binprovider": "env",
-            },
-        )
-        if binary is None:
-            raise ValueError("failed to create binary record")
-
-        if not is_tty:
-            write_record(binary.to_json())
-
-        if created:
-            rprint(f"[green]Created binary: {name} at {abspath}[/green]", file=sys.stderr)
-        else:
-            rprint(f"[dim]Binary already exists: {name} at {abspath}[/dim]", file=sys.stderr)
-
-        return 0
-
-    except Exception as e:
-        rprint(f"[red]Error creating binary: {e}[/red]", file=sys.stderr)
-        return 1
-
-
-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_binaries(
-    name: str | None = None,
-    abspath__icontains: str | None = None,
-    version__icontains: str | None = None,
-    limit: int | None = None,
-) -> int:
-    """
-    List Binaries as JSONL with optional filters.
-
-    Exit codes:
-        0: Success (even if no results)
-    """
-    from archivebox.misc.jsonl import write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    queryset = Binary.objects.all().order_by("name", "-modified_at", "-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "name": name,
-        "abspath__icontains": abspath__icontains,
-        "version__icontains": version__icontains,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for binary in queryset:
-        if is_tty:
-            rprint(f"[cyan]{binary.name:20}[/cyan] [dim]{binary.version:15}[/dim] {binary.abspath}")
-        else:
-            write_record(binary.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} binaries[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_binaries(
-    version: str | None = None,
-    abspath: str | None = None,
-) -> int:
-    """
-    Update Binaries from stdin JSONL.
-
-    Reads Binary records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.machine.models import Binary
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    updated_count = 0
-    for record in records:
-        binary_id = record.get("id")
-        if not binary_id:
-            continue
-
-        try:
-            binary = Binary.objects.get(id=binary_id)
-
-            # Apply updates from CLI flags
-            if version:
-                binary.version = version
-            if abspath:
-                binary.abspath = abspath
-
-            binary.save()
-            updated_count += 1
-
-            if not is_tty:
-                write_record(binary.to_json())
-
-        except Binary.DoesNotExist:
-            rprint(f"[yellow]Binary not found: {binary_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} binaries[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_binaries(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Binaries from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.machine.models import Binary
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    binary_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not binary_ids:
-        rprint("[yellow]No valid binary IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    binaries = Binary.objects.filter(id__in=binary_ids)
-    count = binaries.count()
-
-    if count == 0:
-        rprint("[yellow]No matching binaries found[/yellow]", file=sys.stderr)
-        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} binaries (dry run)[/yellow]", file=sys.stderr)
-        for binary in binaries:
-            rprint(f"  {binary.name} {binary.abspath}", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
-        return 1
-
-    # Perform deletion
-    deleted_count, _ = binaries.delete()
-    rprint(f"[green]Deleted {deleted_count} binaries[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# CLI Commands
-# =============================================================================
-
-
-@click.group()
-def main():
-    """Manage Binary records (detected executables)."""
-    pass
-
-
-@main.command("create")
-@click.option("--name", "-n", required=True, help="Binary name (e.g., chrome, wget)")
-@click.option("--abspath", "-p", required=True, help="Absolute path to binary")
-@click.option("--version", "-v", default="", help="Binary version")
-def create_cmd(name: str, abspath: str, version: str):
-    """Create/register a Binary."""
-    sys.exit(create_binary(name=name, abspath=abspath, version=version))
-
-
-@main.command("list")
-@click.option("--name", "-n", help="Filter by name")
-@click.option("--abspath__icontains", help="Filter by path contains")
-@click.option("--version__icontains", help="Filter by version contains")
-@click.option("--limit", type=int, help="Limit number of results")
-def list_cmd(
-    name: str | None,
-    abspath__icontains: str | None,
-    version__icontains: str | None,
-    limit: int | None,
-):
-    """List Binaries as JSONL."""
-    sys.exit(
-        list_binaries(
-            name=name,
-            abspath__icontains=abspath__icontains,
-            version__icontains=version__icontains,
-            limit=limit,
-        ),
-    )
-
-
-@main.command("update")
-@click.option("--version", "-v", help="Set version")
-@click.option("--abspath", "-p", help="Set path")
-def update_cmd(version: str | None, abspath: str | None):
-    """Update Binaries from stdin JSONL."""
-    sys.exit(update_binaries(version=version, abspath=abspath))
-
-
-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Binaries from stdin JSONL."""
-    sys.exit(delete_binaries(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
-    main()
@@ -1,127 +1,102 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'

 import sys
-import toml
 import rich_click as click
 from rich import print
-from pathlib import Path
+from benedict import benedict

 from archivebox.misc.util import docstring, enforce_types
 from archivebox.misc.toml_util import CustomTOMLEncoder


-def _format_toml(config: dict) -> str:
-    return toml.dumps(config, encoder=CustomTOMLEncoder()).strip().replace("\n\n", "\n")
-
-
@enforce_types
-def config(
-    *keys,
-    get: bool = False,
-    set: bool = False,
-    search: bool = False,
-    reset: bool = False,
-    **kwargs,
-) -> None:
+def config(*keys,
+          get: bool=False,
+          set: bool=False, 
+          search: bool=False,
+          reset: bool=False,
+          **kwargs) -> None:
    """Get and set your ArchiveBox project configuration values"""

    from archivebox.misc.checks import check_data_folder
    from archivebox.misc.logging_util import printable_config
-    from abx_plugins.plugins.base.utils import resolve_alias
-    from archivebox.config.collection import write_config_file
-    from archivebox.config import CONSTANTS_CONFIG
-    from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
-    from archivebox.plugins.discovery import discover_plugin_configs
+    from archivebox.config.collection import load_all_config, write_config_file, get_real_name
+    from archivebox.config.configset import get_flat_config, get_all_configs

    check_data_folder()

-    FLAT_CONFIG = get_config().as_dict()
-    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
-    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
-    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
-    readable_config = {**writable_config, **readonly_config}
+    FLAT_CONFIG = get_flat_config()
    CONFIGS = get_all_configs()
-    plugin_schemas = {
-        plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
-    }
-    core_config_aliases = {
-        alias.upper(): field_name
-        for field_name, field in ArchiveBoxConfig.model_fields.items()
-        for alias in (field_name, str(field.alias or ""))
-        if alias
-    }
-
-    config_options: list[str] = list(kwargs.pop("key=value", []) or keys or [f"{key}={val}" for key, val in kwargs.items()])
+    
+    config_options: list[str] = list(kwargs.pop('key=value', []) or keys or [f'{key}={val}' for key, val in kwargs.items()])
    no_args = not (get or set or reset or config_options)

    matching_config = {}
    if search:
        if config_options:
-            search_terms = [key.strip().lower() for key in config_options]
-
-            for existing_key, value in readable_config.items():
-                if any(term in existing_key.lower() or term in str(value).lower() for term in search_terms):
-                    matching_config[existing_key] = value
-
-            for alias, key in core_config_aliases.items():
-                if key in readable_config and any(term in alias.lower() for term in search_terms):
-                    matching_config[key] = readable_config[key]
-
-            for schema in plugin_schemas.values():
-                for key, metadata in schema.items():
-                    if key in readable_config and any(term in key.lower() or term in str(metadata).lower() for term in search_terms):
-                        matching_config[key] = readable_config[key]
-
+            config_options = [get_real_name(key) for key in config_options]
+            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            for config_section in CONFIGS.values():
+                aliases = getattr(config_section, 'aliases', {})
+                
+                for search_key in config_options:
+                    # search all aliases in the section
+                    for alias_key, key in aliases.items():
+                        if search_key.lower() in alias_key.lower():
+                            matching_config[key] = dict(config_section)[key]
+                    
+                    # search all keys and values in the section
+                    for existing_key, value in dict(config_section).items():
+                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
+                            matching_config[existing_key] = value
+            
        print(printable_config(matching_config))
        raise SystemExit(not matching_config)

    elif get or no_args:
        if config_options:
-            config_options = [
-                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
-            ]
-            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
-            failed_config = [key for key in config_options if key not in readable_config]
+            config_options = [get_real_name(key) for key in config_options]
+            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
            if failed_config:
-                print("\n[red][X] These options failed to get[/red]")
-                print("    {}".format("\n    ".join(config_options)))
+                print('\n[red][X] These options failed to get[/red]')
+                print('    {}'.format('\n    '.join(config_options)))
                raise SystemExit(1)
        else:
-            matching_config = readable_config
+            matching_config = FLAT_CONFIG

        # Display core config sections
        for config_section in CONFIGS.values():
-            section_header = config_section.toml_section_header
-            if isinstance(section_header, str) and section_header:
-                print(f"[grey53]\\[{section_header}][/grey53]")
+            if hasattr(config_section, 'toml_section_header'):
+                print(f'[grey53]\\[{config_section.toml_section_header}][/grey53]')
            else:
-                print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
+                print('[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]')

            kv_in_section = {key: val for key, val in dict(config_section).items() if key in matching_config}
-            print(_format_toml(kv_in_section))
-            print("[grey53]################################################################[/grey53]")
+            print(benedict(kv_in_section).to_toml(encoder=CustomTOMLEncoder()).strip().replace('\n\n', '\n'))
+            print('[grey53]################################################################[/grey53]')

-        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
-        if readonly_keys:
-            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
-            print(_format_toml(readonly_keys))
-            print("[grey53]################################################################[/grey53]")
+        # Display plugin config section
+        from archivebox.hooks import discover_plugin_configs

+        plugin_configs = discover_plugin_configs()
        plugin_keys = {}

        # Collect all plugin config keys
-        for schema in plugin_schemas.values():
-            for key in schema.keys():
-                if key in matching_config and key in writable_config:
+        for plugin_name, schema in plugin_configs.items():
+            if 'properties' not in schema:
+                continue
+            for key in schema['properties'].keys():
+                if key in matching_config:
                    plugin_keys[key] = matching_config[key]

        # Display all plugin config in single [PLUGINS] section
        if plugin_keys:
-            print("[grey53]\\[PLUGINS][/grey53]")
-            print(_format_toml(plugin_keys))
-            print("[grey53]################################################################[/grey53]")
+            print(f'[grey53]\\[PLUGINS][/grey53]')
+            print(benedict(plugin_keys).to_toml(encoder=CustomTOMLEncoder()).strip().replace('\n\n', '\n'))
+            print('[grey53]################################################################[/grey53]')

        raise SystemExit(not matching_config)

@@ -129,22 +104,20 @@ def config(
        new_config = {}
        failed_options = []
        for line in config_options:
-            if line.startswith("#") or not line.strip():
+            if line.startswith('#') or not line.strip():
                continue
-            if "=" not in line:
-                print("[red][X] Config KEY=VALUE must have an = sign in it[/red]")
-                print(f"    {line}")
+            if '=' not in line:
+                print('[red][X] Config KEY=VALUE must have an = sign in it[/red]')
+                print(f'    {line}')
                raise SystemExit(2)

-            raw_key, val = line.split("=", 1)
+            raw_key, val = line.split('=', 1)
            raw_key = raw_key.upper().strip()
-            key = core_config_aliases.get(raw_key) or resolve_alias(raw_key, plugin_schemas)
+            key = get_real_name(raw_key)
            if key != raw_key:
-                print(
-                    f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
-                )
+                print(f'[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]')

-            if key in writable_config:
+            if key in FLAT_CONFIG:
                new_config[key] = val.strip()
            else:
                failed_options.append(line)
@@ -152,7 +125,7 @@ def config(
        if new_config:
            before = FLAT_CONFIG
            matching_config = write_config_file(new_config)
-            after = get_config().as_dict()
+            after = {**load_all_config(), **get_flat_config()}
            print(printable_config(matching_config))

            side_effect_changes = {}
@@ -162,38 +135,38 @@ def config(

            if side_effect_changes:
                print(file=sys.stderr)
-                print("[yellow][i] Note: This change also affected these other options that depended on it:[/yellow]", file=sys.stderr)
-                print("    {}".format(printable_config(side_effect_changes, prefix="    ")), file=sys.stderr)
+                print('[yellow][i] Note: This change also affected these other options that depended on it:[/yellow]', file=sys.stderr)
+                print('    {}'.format(printable_config(side_effect_changes, prefix='    ')), file=sys.stderr)

        if failed_options:
            print()
-            print("[red][X] These options failed to set (check for typos):[/red]")
-            print("    {}".format("\n    ".join(failed_options)))
+            print('[red][X] These options failed to set (check for typos):[/red]')
+            print('    {}'.format('\n    '.join(failed_options)))
            raise SystemExit(1)

    elif reset:
-        print("[red][X] This command is not implemented yet.[/red]")
-        print("    Please manually remove the relevant lines from your config file:")
+        print('[red][X] This command is not implemented yet.[/red]')
+        print('    Please manually remove the relevant lines from your config file:')
        raise SystemExit(2)

    else:
-        print("[red][X] You must pass either --get or --set, or no arguments to get the whole config.[/red]")
-        print("    archivebox config")
-        print("    archivebox config --get SOME_KEY")
-        print("    archivebox config --set SOME_KEY=SOME_VALUE")
+        print('[red][X] You must pass either --get or --set, or no arguments to get the whole config.[/red]')
+        print('    archivebox config')
+        print('    archivebox config --get SOME_KEY')
+        print('    archivebox config --set SOME_KEY=SOME_VALUE')
        raise SystemExit(2)


@click.command()
-@click.option("--search", is_flag=True, help="Search config KEYs, VALUEs, and ALIASES for the given term")
-@click.option("--get", is_flag=True, help="Get the value for the given config KEYs")
-@click.option("--set", is_flag=True, help="Set the given KEY=VALUE config values")
-@click.option("--reset", is_flag=True, help="Reset the given KEY config values to their defaults")
-@click.argument("KEY=VALUE", nargs=-1, type=str)
+@click.option('--search', is_flag=True, help='Search config KEYs, VALUEs, and ALIASES for the given term')
+@click.option('--get', is_flag=True, help='Get the value for the given config KEYs')
+@click.option('--set', is_flag=True, help='Set the given KEY=VALUE config values')
+@click.option('--reset', is_flag=True, help='Reset the given KEY config values to their defaults')
+@click.argument('KEY=VALUE', nargs=-1, type=str)
@docstring(config.__doc__)
 def main(**kwargs) -> None:
    config(**kwargs)


-if __name__ == "__main__":
+if __name__ == '__main__':
    main()
@@ -1,406 +1,192 @@
 #!/usr/bin/env python3

 """
-archivebox crawl <action> [args...] [--filters]
+archivebox crawl [urls...] [--depth=N] [--tag=TAG]

-Manage Crawl records.
+Create Crawl jobs from URLs. Accepts URLs as arguments, from stdin, or via JSONL.
+Does NOT immediately start the crawl - pipe to `archivebox snapshot` to process.

-Actions:
-    create  - Create Crawl jobs from URLs
-    list    - List Crawls as JSONL (with optional filters)
-    update  - Update Crawls from stdin JSONL
-    delete  - Delete Crawls from stdin JSONL
+Input formats:
+    - Plain URLs (one per line)
+    - JSONL: {"url": "...", "depth": 1, "tags": "..."}
+
+Output (JSONL):
+    {"type": "Crawl", "id": "...", "urls": "...", "status": "queued", ...}

 Examples:
-    # Create
-    archivebox crawl create https://example.com https://foo.com --depth=1
-    archivebox crawl create --tag=news https://example.com
+    # Create a crawl job
+    archivebox crawl https://example.com

-    # List with filters
-    archivebox crawl list --status=queued
-    archivebox crawl list --urls__icontains=example.com
+    # Create crawl with depth
+    archivebox crawl --depth=1 https://example.com

-    # Update
-    archivebox crawl list --status=started | archivebox crawl update --status=queued
+    # Full pipeline: create crawl, create snapshots, run extractors
+    archivebox crawl https://example.com | archivebox snapshot | archivebox extract

-    # Delete
-    archivebox crawl list --urls__icontains=spam.com | archivebox crawl delete --yes
-
-    # Full pipeline
-    archivebox crawl create https://example.com | archivebox snapshot create | archivebox run
+    # Process existing Crawl by ID (runs the crawl state machine)
+    archivebox crawl 01234567-89ab-cdef-0123-456789abcdef
 """

-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox crawl'

 import sys
-from collections.abc import Iterable
+from typing import Optional

 import rich_click as click
-from rich import print as rprint
-
-from archivebox.cli.cli_util import apply_filters


-# =============================================================================
-# CREATE
-# =============================================================================
-
-
-def create_crawl(
-    urls: Iterable[str],
+def create_crawls(
+    records: list,
    depth: int = 0,
-    tag: str = "",
-    status: str = "queued",
-    created_by_id: int | None = None,
+    tag: str = '',
+    created_by_id: Optional[int] = None,
 ) -> int:
    """
-    Create a Crawl job from URLs.
+    Create a single Crawl job from all input URLs.

-    Takes URLs as args or stdin, creates one Crawl with all URLs, outputs JSONL.
-    Pass-through: Records that are not URLs are output unchanged (for piping).
+    Takes pre-read records, creates one Crawl with all URLs, outputs JSONL.
+    Does NOT start the crawl - just creates the job in QUEUED state.

    Exit codes:
        0: Success
        1: Failure
    """
-    from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
-    from archivebox.misc.util import validate_url
+    from rich import print as rprint
+
+    from archivebox.misc.jsonl import write_record
    from archivebox.base_models.models import get_or_create_system_user_pk
    from archivebox.crawls.models import Crawl

    created_by_id = created_by_id or get_or_create_system_user_pk()
    is_tty = sys.stdout.isatty()

-    # Collect all input records
-    records = list(read_args_or_stdin(urls))
-
    if not records:
-        rprint("[yellow]No URLs provided. Pass URLs as arguments or via stdin.[/yellow]", file=sys.stderr)
+        rprint('[yellow]No URLs provided. Pass URLs as arguments or via stdin.[/yellow]', file=sys.stderr)
        return 1

-    # Separate pass-through records from URL records
-    url_list = []
-    pass_through_records = []
-
+    # Collect all URLs into a single newline-separated string
+    urls = []
    for record in records:
-        record_type = record.get("type", "")
-
-        # Pass-through: output records that aren't URL/Crawl types
-        if record_type and record_type != TYPE_CRAWL and not record.get("url") and not record.get("urls"):
-            pass_through_records.append(record)
-            continue
-
-        # Handle existing Crawl records (just pass through with id)
-        if record_type == TYPE_CRAWL and record.get("id"):
-            pass_through_records.append(record)
-            continue
-
-        # Collect URLs
-        url = record.get("url")
+        url = record.get('url')
        if url:
-            try:
-                url_list.append(validate_url(str(url)))
-            except ValueError as err:
-                rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
-                return 1
+            urls.append(url)

-        # Handle 'urls' field (newline-separated)
-        urls_field = record.get("urls")
-        if urls_field:
-            for line in urls_field.split("\n"):
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    try:
-                        url_list.append(validate_url(line))
-                    except ValueError as err:
-                        rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
-                        return 1
-
-    # Output pass-through records first
-    if not is_tty:
-        for record in pass_through_records:
-            write_record(record)
-
-    if not url_list:
-        if pass_through_records:
-            # If we had pass-through records but no URLs, that's OK
-            rprint(f"[dim]Passed through {len(pass_through_records)} records, no new URLs[/dim]", file=sys.stderr)
-            return 0
-        rprint("[red]No valid URLs found[/red]", file=sys.stderr)
+    if not urls:
+        rprint('[red]No valid URLs found[/red]', file=sys.stderr)
        return 1

    try:
        # Build crawl record with all URLs as newline-separated string
        crawl_record = {
-            "urls": "\n".join(url_list),
-            "max_depth": depth,
-            "tags_str": tag,
-            "status": status,
-            "label": "",
+            'urls': '\n'.join(urls),
+            'max_depth': depth,
+            'tags_str': tag,
+            'label': '',
        }

-        crawl = Crawl.from_json(crawl_record, overrides={"created_by_id": created_by_id})
+        crawl = Crawl.from_jsonl(crawl_record, overrides={'created_by_id': created_by_id})
        if not crawl:
-            rprint("[red]Failed to create crawl[/red]", file=sys.stderr)
+            rprint('[red]Failed to create crawl[/red]', file=sys.stderr)
            return 1

        # Output JSONL record (only when piped)
        if not is_tty:
-            write_record(crawl.to_json())
+            write_record(crawl.to_jsonl())

-        rprint(f"[green]Created crawl with {len(url_list)} URLs[/green]", file=sys.stderr)
+        rprint(f'[green]Created crawl with {len(urls)} URLs[/green]', file=sys.stderr)

        # If TTY, show human-readable output
        if is_tty:
-            rprint(f"  [dim]{crawl.id}[/dim]", file=sys.stderr)
-            for url in url_list[:5]:  # Show first 5 URLs
-                rprint(f"    {url[:70]}", file=sys.stderr)
-            if len(url_list) > 5:
-                rprint(f"    ... and {len(url_list) - 5} more", file=sys.stderr)
+            rprint(f'  [dim]{crawl.id}[/dim]', file=sys.stderr)
+            for url in urls[:5]:  # Show first 5 URLs
+                rprint(f'    {url[:70]}', file=sys.stderr)
+            if len(urls) > 5:
+                rprint(f'    ... and {len(urls) - 5} more', file=sys.stderr)

        return 0

    except Exception as e:
-        rprint(f"[red]Error creating crawl: {e}[/red]", file=sys.stderr)
+        rprint(f'[red]Error creating crawl: {e}[/red]', file=sys.stderr)
        return 1


-# =============================================================================
-# LIST
-# =============================================================================
-
-
-def list_crawls(
-    status: str | None = None,
-    urls__icontains: str | None = None,
-    max_depth: int | None = None,
-    limit: int | None = None,
-) -> int:
+def process_crawl_by_id(crawl_id: str) -> int:
    """
-    List Crawls as JSONL with optional filters.
+    Process a single Crawl by ID (used by workers).

-    Exit codes:
-        0: Success (even if no results)
+    Triggers the Crawl's state machine tick() which will:
+    - Transition from queued -> started (creates root snapshot)
+    - Transition from started -> sealed (when all snapshots done)
    """
-    from archivebox.misc.jsonl import write_record
+    from rich import print as rprint
    from archivebox.crawls.models import Crawl

-    is_tty = sys.stdout.isatty()
-
-    queryset = Crawl.objects.all().order_by("-created_at")
-
-    # Apply filters
-    filter_kwargs = {
-        "status": status,
-        "urls__icontains": urls__icontains,
-        "max_depth": max_depth,
-    }
-    queryset = apply_filters(queryset, filter_kwargs, limit=limit)
-
-    count = 0
-    for crawl in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(crawl.status, "dim")
-            url_preview = crawl.urls[:50].replace("\n", " ")
-            rprint(f"[{status_color}]{crawl.status:8}[/{status_color}] [dim]{crawl.id}[/dim] {url_preview}...")
-        else:
-            write_record(crawl.to_json())
-        count += 1
-
-    rprint(f"[dim]Listed {count} crawls[/dim]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# UPDATE
-# =============================================================================
-
-
-def update_crawls(
-    status: str | None = None,
-    max_depth: int | None = None,
-) -> int:
-    """
-    Update Crawls from stdin JSONL.
-
-    Reads Crawl records from stdin and applies updates.
-    Uses PATCH semantics - only specified fields are updated.
-
-    Exit codes:
-        0: Success
-        1: No input or error
-    """
-    from django.utils import timezone
-
-    from archivebox.misc.jsonl import read_stdin, write_record
-    from archivebox.crawls.models import Crawl
-
-    is_tty = sys.stdout.isatty()
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
+    try:
+        crawl = Crawl.objects.get(id=crawl_id)
+    except Crawl.DoesNotExist:
+        rprint(f'[red]Crawl {crawl_id} not found[/red]', file=sys.stderr)
        return 1

-    updated_count = 0
-    for record in records:
-        crawl_id = record.get("id")
-        if not crawl_id:
-            continue
+    rprint(f'[blue]Processing Crawl {crawl.id} (status={crawl.status})[/blue]', file=sys.stderr)

-        try:
-            crawl = Crawl.objects.get(id=crawl_id)
-
-            if status:
-                if status not in Crawl.StatusChoices.values:
-                    rprint(f"[red]Invalid crawl status: {status}[/red]", file=sys.stderr)
-                    continue
-                if status == Crawl.StatusChoices.SEALED:
-                    crawl.cancel()
-                elif status == Crawl.StatusChoices.PAUSED:
-                    crawl.pause()
-                elif status == Crawl.StatusChoices.QUEUED:
-                    if crawl.status == Crawl.StatusChoices.PAUSED:
-                        crawl.resume()
-                    else:
-                        crawl.update_and_requeue(status=Crawl.StatusChoices.QUEUED, retry_at=timezone.now())
-                elif status == Crawl.StatusChoices.STARTED:
-                    crawl.update_and_requeue(status=Crawl.StatusChoices.STARTED, retry_at=timezone.now())
-            if max_depth is not None:
-                crawl.safe_update({"max_depth": max_depth, "modified_at": timezone.now()}, refresh=False)
-                crawl.max_depth = max_depth
-            elif not status:
-                crawl.safe_update({"modified_at": timezone.now()}, refresh=False)
-            updated_count += 1
-
-            if not is_tty:
-                crawl.refresh_from_db()
-                write_record(crawl.to_json())
-
-        except Crawl.DoesNotExist:
-            rprint(f"[yellow]Crawl not found: {crawl_id}[/yellow]", file=sys.stderr)
-            continue
-
-    rprint(f"[green]Updated {updated_count} crawls[/green]", file=sys.stderr)
-    return 0
-
-
-# =============================================================================
-# DELETE
-# =============================================================================
-
-
-def delete_crawls(yes: bool = False, dry_run: bool = False) -> int:
-    """
-    Delete Crawls from stdin JSONL.
-
-    Requires --yes flag to confirm deletion.
-
-    Exit codes:
-        0: Success
-        1: No input or missing --yes flag
-    """
-    from archivebox.misc.jsonl import read_stdin
-    from archivebox.crawls.models import Crawl
-
-    records = list(read_stdin())
-    if not records:
-        rprint("[yellow]No records provided via stdin[/yellow]", file=sys.stderr)
-        return 1
-
-    crawl_ids = [r.get("id") for r in records if r.get("id")]
-
-    if not crawl_ids:
-        rprint("[yellow]No valid crawl IDs in input[/yellow]", file=sys.stderr)
-        return 1
-
-    crawls = Crawl.objects.filter(id__in=crawl_ids)
-    count = crawls.count()
-
-    if count == 0:
-        rprint("[yellow]No matching crawls found[/yellow]", file=sys.stderr)
+    try:
+        crawl.sm.tick()
+        crawl.refresh_from_db()
+        rprint(f'[green]Crawl complete (status={crawl.status})[/green]', file=sys.stderr)
        return 0
-
-    if dry_run:
-        rprint(f"[yellow]Would delete {count} crawls (dry run)[/yellow]", file=sys.stderr)
-        for crawl in crawls:
-            url_preview = crawl.urls[:50].replace("\n", " ")
-            rprint(f"  [dim]{crawl.id}[/dim] {url_preview}...", file=sys.stderr)
-        return 0
-
-    if not yes:
-        rprint("[red]Use --yes to confirm deletion[/red]", file=sys.stderr)
+    except Exception as e:
+        rprint(f'[red]Crawl error: {type(e).__name__}: {e}[/red]', file=sys.stderr)
        return 1

-    # Perform deletion
-    deleted_count, _ = crawls.delete()
-    rprint(f"[green]Deleted {deleted_count} crawls[/green]", file=sys.stderr)
-    return 0
+
+def is_crawl_id(value: str) -> bool:
+    """Check if value looks like a Crawl UUID."""
+    import re
+    uuid_pattern = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', re.I)
+    if not uuid_pattern.match(value):
+        return False
+    # Verify it's actually a Crawl (not a Snapshot or other object)
+    from archivebox.crawls.models import Crawl
+    return Crawl.objects.filter(id=value).exists()


-# =============================================================================
-# CLI Commands
-# =============================================================================
+@click.command()
+@click.option('--depth', '-d', type=int, default=0, help='Max depth for recursive crawling (default: 0, no recursion)')
+@click.option('--tag', '-t', default='', help='Comma-separated tags to add to snapshots')
+@click.argument('args', nargs=-1)
+def main(depth: int, tag: str, args: tuple):
+    """Create Crawl jobs from URLs, or process existing Crawls by ID"""
+    from archivebox.misc.jsonl import read_args_or_stdin

+    # Read all input
+    records = list(read_args_or_stdin(args))

-@click.group()
-def main():
-    """Manage Crawl records."""
-    pass
+    if not records:
+        from rich import print as rprint
+        rprint('[yellow]No URLs or Crawl IDs provided. Pass as arguments or via stdin.[/yellow]', file=sys.stderr)
+        sys.exit(1)

-
-@main.command("create")
-@click.argument("urls", nargs=-1)
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-def create_cmd(urls: tuple, depth: int, tag: str, status: str):
-    """Create a Crawl job from URLs or stdin."""
-    sys.exit(create_crawl(urls, depth=depth, tag=tag, status=status))
-
-
-@main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--urls__icontains", help="Filter by URLs contains")
-@click.option("--max-depth", type=int, help="Filter by max depth")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-def list_cmd(
-    status: str | None,
-    urls__icontains: str | None,
-    max_depth: int | None,
-    limit: int | None,
-):
-    """List Crawls as JSONL."""
-    sys.exit(
-        list_crawls(
-            status=status,
-            urls__icontains=urls__icontains,
-            max_depth=max_depth,
-            limit=limit,
-        ),
+    # Check if input looks like existing Crawl IDs to process
+    # If ALL inputs are Crawl UUIDs, process them
+    all_are_crawl_ids = all(
+        is_crawl_id(r.get('id') or r.get('url', ''))
+        for r in records
    )

-
-@main.command("update")
-@click.option("--status", "-s", help="Set status")
-@click.option("--max-depth", type=int, help="Set max depth")
-def update_cmd(status: str | None, max_depth: int | None):
-    """Update Crawls from stdin JSONL."""
-    sys.exit(update_crawls(status=status, max_depth=max_depth))
+    if all_are_crawl_ids:
+        # Process existing Crawls by ID
+        exit_code = 0
+        for record in records:
+            crawl_id = record.get('id') or record.get('url')
+            result = process_crawl_by_id(crawl_id)
+            if result != 0:
+                exit_code = result
+        sys.exit(exit_code)
+    else:
+        # Default behavior: create Crawl jobs from URLs
+        sys.exit(create_crawls(records, depth=depth, tag=tag))


-@main.command("delete")
-@click.option("--yes", "-y", is_flag=True, help="Confirm deletion")
-@click.option("--dry-run", is_flag=True, help="Show what would be deleted")
-def delete_cmd(yes: bool, dry_run: bool):
-    """Delete Crawls from stdin JSONL."""
-    sys.exit(delete_crawls(yes=yes, dry_run=dry_run))
-
-
-if __name__ == "__main__":
+if __name__ == '__main__':
    main()
@@ -1,203 +1,265 @@
 #!/usr/bin/env python3

-"""Run abx-dl snapshot hooks for existing ArchiveBox snapshots."""
+"""
+archivebox extract [snapshot_ids...] [--plugins=NAMES]

-__package__ = "archivebox.cli"
-__command__ = "archivebox extract"
+Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL.
+
+Input formats:
+    - Snapshot UUIDs (one per line)
+    - JSONL: {"type": "Snapshot", "id": "...", "url": "..."}
+    - JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."}
+
+Output (JSONL):
+    {"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."}
+
+Examples:
+    # Extract specific snapshot
+    archivebox extract 01234567-89ab-cdef-0123-456789abcdef
+
+    # Pipe from snapshot command
+    archivebox snapshot https://example.com | archivebox extract
+
+    # Run specific plugins only
+    archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef
+
+    # Chain commands
+    archivebox crawl https://example.com | archivebox snapshot | archivebox extract
+"""
+
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox extract'

 import sys
-from collections import defaultdict
-from contextlib import redirect_stdout
+from typing import Optional, List

 import rich_click as click


-def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[str]], set[str]]:
-    """Resolve CLI records to snapshot-level execution requests.
-
-    ArchiveResult input is accepted as a convenient reference to its parent
-    Snapshot and plugin. It is never reset or converted into queued work.
+def process_archiveresult_by_id(archiveresult_id: str) -> int:
    """
-    from archivebox.api.v1_core import _uuid_ref_query
-    from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.misc.jsonl import TYPE_ARCHIVERESULT
+    Run extraction for a single ArchiveResult by ID (used by workers).

-    explicit_plugins = {name.strip() for name in plugins.split(",") if name.strip()}
-    requested: dict[str, set[str]] = defaultdict(set)
-    missing: set[str] = set()
-
-    for record in records:
-        record_type = record.get("type")
-        record_id = str(record.get("id") or "")
-
-        if record_type == TYPE_ARCHIVERESULT and record.get("snapshot_id"):
-            requested[str(record["snapshot_id"])].update(explicit_plugins or {str(record.get("plugin") or "")})
-            requested[str(record["snapshot_id"])].discard("")
-            continue
-
-        if record_type == TYPE_ARCHIVERESULT:
-            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
-            if result is not None:
-                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
-                continue
-
-        snapshot_id = str(record.get("snapshot_id") or record_id or "")
-        snapshot = Snapshot.objects.filter(id=snapshot_id).only("id").first() if snapshot_id else None
-        if snapshot is None and record.get("url"):
-            snapshot = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").only("id").first()
-        # Bare UUID CLI arguments are parsed as Snapshot records because the
-        # input layer cannot know which model owns them. Preserve the public
-        # convenience of passing an ArchiveResult ID by trying that reference
-        # only after the Snapshot lookup misses.
-        if snapshot is None and record_id and not record.get("url"):
-            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
-            if result is not None:
-                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
-                continue
-        if snapshot is None:
-            missing.add(snapshot_id or str(record.get("url") or ""))
-            continue
-        requested[str(snapshot.id)].update(explicit_plugins)
-
-    return requested, missing
-
-
-def _run_snapshot_requests(requested: dict[str, set[str]], *, wait: bool, show_progress: bool) -> int:
-    from django.utils import timezone
+    Triggers the ArchiveResult's state machine tick() to run the extractor plugin.
+    """
    from rich import print as rprint
+    from archivebox.core.models import ArchiveResult

-    from archivebox.core.models import Snapshot
-    from archivebox.services.runner import run_crawl
-
-    snapshots = {str(snapshot.id): snapshot for snapshot in Snapshot.objects.filter(id__in=requested).select_related("crawl")}
-    if not snapshots:
-        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
+    try:
+        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
+    except ArchiveResult.DoesNotExist:
+        rprint(f'[red]ArchiveResult {archiveresult_id} not found[/red]', file=sys.stderr)
        return 1

-    from archivebox.config.common import get_config
-    from archivebox.plugins.discovery import get_enabled_plugins
+    rprint(f'[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]', file=sys.stderr)

-    for snapshot_id, plugin_names in requested.items():
-        snapshot = snapshots.get(snapshot_id)
-        if snapshot is not None and not plugin_names:
-            plugin_names.update(get_enabled_plugins(config=get_config(crawl=snapshot.crawl, snapshot=snapshot)))
+    try:
+        # Trigger state machine tick - this runs the actual extraction
+        archiveresult.sm.tick()
+        archiveresult.refresh_from_db()

-    if wait and any("search_backend_sonic" in plugin_names for plugin_names in requested.values()):
-        from archivebox.core.takeover_util import ensure_daemon_stack
+        if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
+            print(f'[green]Extraction succeeded: {archiveresult.output_str}[/green]')
+            return 0
+        elif archiveresult.status == ArchiveResult.StatusChoices.FAILED:
+            print(f'[red]Extraction failed: {archiveresult.output_str}[/red]', file=sys.stderr)
+            return 1
+        else:
+            # Still in progress or backoff - not a failure
+            print(f'[yellow]Extraction status: {archiveresult.status}[/yellow]')
+            return 0

-        with redirect_stdout(sys.stderr):
-            ensure_daemon_stack(reason="Sonic snapshot indexing")
-
-    # Explicit extraction resumes open/paused snapshots at the snapshot level.
-    # Sealed snapshots stay sealed during targeted maintenance backfills.
-    if wait:
-        for snapshot in snapshots.values():
-            if snapshot.status != Snapshot.StatusChoices.SEALED:
-                snapshot.update_and_requeue(
-                    status=Snapshot.StatusChoices.QUEUED,
-                    retry_at=timezone.now(),
-                )
-
-    if not wait:
-        for snapshot_id, plugin_names in requested.items():
-            snapshot = snapshots.get(snapshot_id)
-            if snapshot is None:
-                continue
-            if plugin_names:
-                snapshot.schedule_plugin_run(plugin_names, when=timezone.now())
-            elif snapshot.status == Snapshot.StatusChoices.SEALED:
-                snapshot.update_and_requeue(retry_at=timezone.now())
-            else:
-                snapshot.update_and_requeue(
-                    status=Snapshot.StatusChoices.QUEUED,
-                    retry_at=timezone.now(),
-                )
-        if show_progress:
-            rprint(f"[blue]Queued {len(snapshots)} snapshots for extraction[/blue]", file=sys.stderr)
-        return 0
-
-    grouped: dict[tuple[str, tuple[str, ...]], list[str]] = defaultdict(list)
-    for snapshot_id, plugin_names in requested.items():
-        snapshot = snapshots.get(snapshot_id)
-        if snapshot is not None:
-            grouped[(str(snapshot.crawl_id), tuple(sorted(plugin_names)))].append(snapshot_id)
-
-    for (crawl_id, plugin_names), snapshot_ids in grouped.items():
-        run_crawl(
-            crawl_id,
-            snapshot_ids=sorted(snapshot_ids),
-            selected_plugins=list(plugin_names) or None,
-            show_progress=show_progress,
-        )
-    return 0
+    except Exception as e:
+        print(f'[red]Extraction error: {type(e).__name__}: {e}[/red]', file=sys.stderr)
+        return 1


 def run_plugins(
    args: tuple,
-    records: list[dict] | None = None,
-    plugins: str = "",
+    plugins: str = '',
    wait: bool = True,
-    emit_results: bool = True,
-    show_progress: bool = True,
 ) -> int:
-    """Execute selected plugins through the snapshot-level runner."""
+    """
+    Run plugins on Snapshots from input.
+
+    Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
    from rich import print as rprint
+    from django.utils import timezone

-    from archivebox.core.models import Snapshot
-    from archivebox.misc.jsonl import read_args_or_stdin, write_record
-
-    if records is None:
-        records = list(read_args_or_stdin(args))
-    if not records:
-        rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
-        return 1
-
-    requested, missing = _resolve_requests(records, plugins)
-    for value in sorted(missing):
-        rprint(f"[yellow]Snapshot or ArchiveResult not found: {value}[/yellow]", file=sys.stderr)
-    if not requested:
-        return 1
-
-    exit_code = _run_snapshot_requests(requested, wait=wait, show_progress=show_progress)
-    if exit_code or not emit_results:
-        return exit_code
+    from archivebox.misc.jsonl import (
+        read_args_or_stdin, write_record,
+        TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
+    )
+    from archivebox.core.models import Snapshot, ArchiveResult
+    from archivebox.workers.orchestrator import Orchestrator

    is_tty = sys.stdout.isatty()
-    for snapshot in Snapshot.objects.filter(id__in=requested):
-        results = snapshot.archiveresult_set.all()
-        requested_plugins = requested[str(snapshot.id)]
-        if requested_plugins:
-            results = results.filter(plugin__in=requested_plugins)
-        for result in results:
-            if is_tty:
-                color = {"succeeded": "green", "failed": "red", "skipped": "yellow"}.get(result.status, "dim")
-                rprint(f"  [{color}]{result.status}[/{color}] {result.plugin} → {result.output_str or ''}", file=sys.stderr)
-            else:
-                write_record(result.to_json())
+
+    # Parse comma-separated plugins list once (reused in creation and filtering)
+    plugins_list = [p.strip() for p in plugins.split(',') if p.strip()] if plugins else []
+
+    # Collect all input records
+    records = list(read_args_or_stdin(args))
+
+    if not records:
+        rprint('[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]', file=sys.stderr)
+        return 1
+
+    # Gather snapshot IDs to process
+    snapshot_ids = set()
+    for record in records:
+        record_type = record.get('type')
+
+        if record_type == TYPE_SNAPSHOT:
+            snapshot_id = record.get('id')
+            if snapshot_id:
+                snapshot_ids.add(snapshot_id)
+            elif record.get('url'):
+                # Look up by URL (get most recent if multiple exist)
+                snap = Snapshot.objects.filter(url=record['url']).order_by('-created_at').first()
+                if snap:
+                    snapshot_ids.add(str(snap.id))
+                else:
+                    rprint(f'[yellow]Snapshot not found for URL: {record["url"]}[/yellow]', file=sys.stderr)
+
+        elif record_type == TYPE_ARCHIVERESULT:
+            snapshot_id = record.get('snapshot_id')
+            if snapshot_id:
+                snapshot_ids.add(snapshot_id)
+
+        elif 'id' in record:
+            # Assume it's a snapshot ID
+            snapshot_ids.add(record['id'])
+
+    if not snapshot_ids:
+        rprint('[red]No valid snapshot IDs found in input[/red]', file=sys.stderr)
+        return 1
+
+    # Get snapshots and ensure they have pending ArchiveResults
+    processed_count = 0
+    for snapshot_id in snapshot_ids:
+        try:
+            snapshot = Snapshot.objects.get(id=snapshot_id)
+        except Snapshot.DoesNotExist:
+            rprint(f'[yellow]Snapshot {snapshot_id} not found[/yellow]', file=sys.stderr)
+            continue
+
+        # Create pending ArchiveResults if needed
+        if plugins_list:
+            # Only create for specific plugins
+            for plugin_name in plugins_list:
+                result, created = ArchiveResult.objects.get_or_create(
+                    snapshot=snapshot,
+                    plugin=plugin_name,
+                    defaults={
+                        'status': ArchiveResult.StatusChoices.QUEUED,
+                        'retry_at': timezone.now(),
+                    }
+                )
+                if not created and result.status in [ArchiveResult.StatusChoices.FAILED, ArchiveResult.StatusChoices.SKIPPED]:
+                    # Reset for retry
+                    result.status = ArchiveResult.StatusChoices.QUEUED
+                    result.retry_at = timezone.now()
+                    result.save()
+        else:
+            # Create all pending plugins
+            snapshot.create_pending_archiveresults()
+
+        # Reset snapshot status to allow processing
+        if snapshot.status == Snapshot.StatusChoices.SEALED:
+            snapshot.status = Snapshot.StatusChoices.STARTED
+            snapshot.retry_at = timezone.now()
+            snapshot.save()
+
+        processed_count += 1
+
+    if processed_count == 0:
+        rprint('[red]No snapshots to process[/red]', file=sys.stderr)
+        return 1
+
+    rprint(f'[blue]Queued {processed_count} snapshots for extraction[/blue]', file=sys.stderr)
+
+    # Run orchestrator if --wait (default)
+    if wait:
+        rprint('[blue]Running plugins...[/blue]', file=sys.stderr)
+        orchestrator = Orchestrator(exit_on_idle=True)
+        orchestrator.runloop()
+
+    # Output results as JSONL (when piped) or human-readable (when TTY)
+    for snapshot_id in snapshot_ids:
+        try:
+            snapshot = Snapshot.objects.get(id=snapshot_id)
+            results = snapshot.archiveresult_set.all()
+            if plugins_list:
+                results = results.filter(plugin__in=plugins_list)
+
+            for result in results:
+                if is_tty:
+                    status_color = {
+                        'succeeded': 'green',
+                        'failed': 'red',
+                        'skipped': 'yellow',
+                    }.get(result.status, 'dim')
+                    rprint(f'  [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ""}', file=sys.stderr)
+                else:
+                    write_record(result.to_jsonl())
+        except Snapshot.DoesNotExist:
+            continue
+
    return 0


-def process_archiveresult_by_id(archiveresult_id: str) -> int:
-    """Re-run the parent Snapshot plugin referenced by an ArchiveResult."""
-    return run_plugins((), records=[{"id": archiveresult_id}], wait=True)
+def is_archiveresult_id(value: str) -> bool:
+    """Check if value looks like an ArchiveResult UUID."""
+    import re
+    uuid_pattern = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', re.I)
+    if not uuid_pattern.match(value):
+        return False
+    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
+    from archivebox.core.models import ArchiveResult
+    return ArchiveResult.objects.filter(id=value).exists()


@click.command()
-@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run")
-@click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
-@click.argument("args", nargs=-1)
+@click.option('--plugins', '-p', default='', help='Comma-separated list of plugins to run (e.g., screenshot,singlefile)')
+@click.option('--wait/--no-wait', default=True, help='Wait for plugins to complete (default: wait)')
+@click.argument('args', nargs=-1)
 def main(plugins: str, wait: bool, args: tuple):
-    """Run plugins on Snapshots; ArchiveResult IDs select their parent plugin."""
+    """Run plugins on Snapshots, or process existing ArchiveResults by ID"""
    from archivebox.misc.jsonl import read_args_or_stdin

+    # Read all input
    records = list(read_args_or_stdin(args))
+
    if not records:
        from rich import print as rprint
-
-        rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
+        rprint('[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]', file=sys.stderr)
        sys.exit(1)
-    sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
+
+    # Check if input looks like existing ArchiveResult IDs to process
+    all_are_archiveresult_ids = all(
+        is_archiveresult_id(r.get('id') or r.get('url', ''))
+        for r in records
+    )
+
+    if all_are_archiveresult_ids:
+        # Process existing ArchiveResults by ID
+        exit_code = 0
+        for record in records:
+            archiveresult_id = record.get('id') or record.get('url')
+            result = process_archiveresult_by_id(archiveresult_id)
+            if result != 0:
+                exit_code = result
+        sys.exit(exit_code)
+    else:
+        # Default behavior: run plugins on Snapshots from input
+        sys.exit(run_plugins(args, plugins=plugins, wait=wait))


-if __name__ == "__main__":
+if __name__ == '__main__':
    main()
@@ -1,12 +1,8 @@
 #!/usr/bin/env python3
-__package__ = "archivebox.cli"
-__command__ = "archivebox help"
+__package__ = 'archivebox.cli'
+__command__ = 'archivebox help'

-import ast
-import importlib.util
-import os
-import sys
-from datetime import UTC, datetime
+import os    
 from pathlib import Path

 import click
@@ -14,74 +10,40 @@ from rich import print
 from rich.panel import Panel


-def _command_doc(cmd: str, import_path: str) -> str:
-    def first_doc_line(docstring: str | None) -> str:
-        lines = (docstring or "").splitlines()
-        return lines[0] if lines else ""
-
-    modname, _ = import_path.rsplit(".", 1)
-    spec = importlib.util.find_spec(modname)
-    if spec is None or spec.origin is None:
-        return ""
-
-    try:
-        tree = ast.parse(Path(spec.origin).read_text())
-    except OSError:
-        return ""
-
-    for name in (cmd, "main"):
-        for node in tree.body:
-            if isinstance(node, ast.FunctionDef) and node.name == name:
-                return first_doc_line(ast.get_docstring(node))
-
-    return first_doc_line(ast.get_docstring(tree))
-
-
 def help() -> None:
    """Print the ArchiveBox help message and usage"""

    from archivebox.cli import ArchiveBoxGroup
    from archivebox.config import CONSTANTS
    from archivebox.config.permissions import IN_DOCKER
-
-    version_msg = (
-        f"[dark_magenta]\\[{datetime.now(UTC):%Y-%m-%d %H:%M:%S}][/dark_magenta] "
-        f"[dark_red]ArchiveBox[/dark_red] [dark_goldenrod]v{CONSTANTS.VERSION}[/dark_goldenrod]: "
-        "[green4]archivebox [green3]help"
+    from archivebox.misc.logging_util import log_cli_command
+    
+    log_cli_command('help', [], None, '.')
+    
+    COMMANDS_HELP_TEXT = '\n    '.join(
+        f'[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}'
+        for cmd in ArchiveBoxGroup.meta_commands.keys()
+    ) + '\n\n    ' + '\n    '.join(
+        f'[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}'
+        for cmd in ArchiveBoxGroup.setup_commands.keys()
+    ) + '\n\n    ' + '\n    '.join(
+        f'[green]{cmd.ljust(20)}[/green] {ArchiveBoxGroup._lazy_load(cmd).__doc__}'
+        for cmd in ArchiveBoxGroup.archive_commands.keys()
    )
-    print(Panel(version_msg), file=sys.stderr)
-
-    COMMANDS_HELP_TEXT = (
-        "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.meta_commands.items())
-        + "\n\n    "
-        + "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.setup_commands.items())
-        + "\n\n    "
-        + "\n    ".join(
-            f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.archive_commands.items()
-        )
-    )
-
-    DOCKER_USAGE = (
-        """
+    
+    DOCKER_USAGE = '''
 [dodger_blue3]Docker Usage:[/dodger_blue3]
    [grey53]# using Docker Compose:[/grey53]
    [blue]docker compose run[/blue] [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]

    [grey53]# using Docker:[/grey53]
-    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 5797:5797[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
-"""
-        if IN_DOCKER
-        else ""
-    )
-    DOCKER_DOCS = (
-        "\n    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker[/link]"
-        if IN_DOCKER
-        else ""
-    )
-    DOCKER_OUTSIDE_HINT = "\n    [grey53]# outside of Docker:[/grey53]" if IN_DOCKER else ""
-    DOCKER_CMD_PREFIX = "[blue]docker ... [/blue]" if IN_DOCKER else ""
+    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 8000:8000[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
+''' if IN_DOCKER else ''
+    DOCKER_DOCS = '\n    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker[/link]' if IN_DOCKER else ''
+    DOCKER_OUTSIDE_HINT = "\n    [grey53]# outside of Docker:[/grey53]" if IN_DOCKER else ''
+    DOCKER_CMD_PREFIX = "[blue]docker ... [/blue]" if IN_DOCKER else ''

-    print(f"""{DOCKER_USAGE}
+    print(f'''{DOCKER_USAGE}
 [deep_sky_blue4]Usage:[/deep_sky_blue4]{DOCKER_OUTSIDE_HINT}
    [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]

@@ -92,11 +54,12 @@ def help() -> None:
    [link=https://github.com/ArchiveBox/ArchiveBox/wiki]https://github.com/ArchiveBox/ArchiveBox/wiki[/link]{DOCKER_DOCS}
    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage[/link]
    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
-""")
-
+''')
+    
+    
    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
-        pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
-        EXAMPLE_USAGE = f"""
+        pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path('~').expanduser()), '~')
+        EXAMPLE_USAGE = f'''
 [light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]

 [violet]Hint:[/violet] [i]Common maintenance tasks:[/i]
@@ -109,50 +72,34 @@ def help() -> None:
    [dark_green]archivebox[/dark_green] [green]add[/green] --depth=1 "https://example.com/some/page"
    [dark_green]archivebox[/dark_green] [green]list[/green] --sort=timestamp --csv=timestamp,downloaded_at,url,title
    [dark_green]archivebox[/dark_green] [green]schedule[/green] --every=day --depth=1 "https://example.com/some/feed.rss"
-    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:5797[/blue]                [grey53]# Start the Web UI / API server[/grey53]
-"""
-        print(
-            Panel(
-                EXAMPLE_USAGE,
-                expand=False,
-                border_style="grey53",
-                title="[green3]:white_check_mark: A collection [light_slate_blue]DATA DIR[/light_slate_blue] is currently active[/green3]",
-                subtitle="Commands run inside this dir will only apply to this collection.",
-            ),
-        )
+    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:8000[/blue]                [grey53]# Start the Web UI / API server[/grey53]
+'''
+        print(Panel(EXAMPLE_USAGE, expand=False, border_style='grey53', title='[green3]:white_check_mark: A collection [light_slate_blue]DATA DIR[/light_slate_blue] is currently active[/green3]', subtitle='Commands run inside this dir will only apply to this collection.'))
    else:
-        DATA_SETUP_HELP = "\n"
+        DATA_SETUP_HELP = '\n'
        if IN_DOCKER:
-            DATA_SETUP_HELP += "[violet]Hint:[/violet] When using Docker, you need to mount a volume to use as your data dir:\n"
-            DATA_SETUP_HELP += "    docker run [violet]-v /some/path/data:/data[/violet] archivebox/archivebox ...\n\n"
-        DATA_SETUP_HELP += "To load an [dark_blue]existing[/dark_blue] collection:\n"
-        DATA_SETUP_HELP += "    1. [green]cd[/green] ~/archivebox/data     [grey53]# go into existing [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
-        DATA_SETUP_HELP += f"    2. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# migrate to latest version (safe to run multiple times)[/grey53]\n"
-        DATA_SETUP_HELP += f"    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-update all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
-        DATA_SETUP_HELP += f"    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ...get help with next steps... [/grey53]\n\n"
-        DATA_SETUP_HELP += "To start a [sea_green1]new[/sea_green1] collection:\n"
-        DATA_SETUP_HELP += "    1. [green]mkdir[/green] ~/archivebox/data  [grey53]# create a new, empty [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n"
-        DATA_SETUP_HELP += "    2. [green]cd[/green] ~/archivebox/data     [grey53]# cd into the new directory[/grey53]\n"
-        DATA_SETUP_HELP += f"    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# initialize ArchiveBox in the new data dir[/grey53]\n"
-        DATA_SETUP_HELP += f"    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-install all plugins (wget, chrome, singlefile, etc.)[/grey53]\n"
-        DATA_SETUP_HELP += f"    5. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ... get help with next steps... [/grey53]\n"
-        print(
-            Panel(
-                DATA_SETUP_HELP,
-                expand=False,
-                border_style="grey53",
-                title="[red]:cross_mark: No collection is currently active[/red]",
-                subtitle="All archivebox [green]commands[/green] should be run from inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
-            ),
-        )
+            DATA_SETUP_HELP += '[violet]Hint:[/violet] When using Docker, you need to mount a volume to use as your data dir:\n'
+            DATA_SETUP_HELP += '    docker run [violet]-v /some/path/data:/data[/violet] archivebox/archivebox ...\n\n'
+        DATA_SETUP_HELP += 'To load an [dark_blue]existing[/dark_blue] collection:\n'
+        DATA_SETUP_HELP += '    1. [green]cd[/green] ~/archivebox/data     [grey53]# go into existing [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n'
+        DATA_SETUP_HELP += f'    2. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# migrate to latest version (safe to run multiple times)[/grey53]\n'
+        DATA_SETUP_HELP += f'    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-update all plugins (wget, chrome, singlefile, etc.)[/grey53]\n'
+        DATA_SETUP_HELP += f'    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ...get help with next steps... [/grey53]\n\n'
+        DATA_SETUP_HELP += 'To start a [sea_green1]new[/sea_green1] collection:\n'
+        DATA_SETUP_HELP += '    1. [green]mkdir[/green] ~/archivebox/data  [grey53]# create a new, empty [light_slate_blue]DATA DIR[/light_slate_blue] (can be anywhere)[/grey53]\n'
+        DATA_SETUP_HELP += '    2. [green]cd[/green] ~/archivebox/data     [grey53]# cd into the new directory[/grey53]\n'
+        DATA_SETUP_HELP += f'    3. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]init[/green]          [grey53]# initialize ArchiveBox in the new data dir[/grey53]\n'
+        DATA_SETUP_HELP += f'    4. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]install[/green]       [grey53]# auto-install all plugins (wget, chrome, singlefile, etc.)[/grey53]\n'
+        DATA_SETUP_HELP += f'    5. {DOCKER_CMD_PREFIX}[dark_green]archivebox[/dark_green] [green]help[/green]          [grey53]# ... get help with next steps... [/grey53]\n'
+        print(Panel(DATA_SETUP_HELP, expand=False, border_style='grey53', title='[red]:cross_mark: No collection is currently active[/red]', subtitle='All archivebox [green]commands[/green] should be run from inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]'))
+


@click.command()
-@click.option("--help", "-h", is_flag=True, help="Show help")
+@click.option('--help', '-h', is_flag=True, help='Show help')
 def main(**kwargs):
    """Print the ArchiveBox help message and usage"""
    return help()

-
-if __name__ == "__main__":
+if __name__ == '__main__':
    main()
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'

 import os
 import sys
@@ -12,204 +12,184 @@ import rich_click as click
 from archivebox.misc.util import docstring, enforce_types


-def _display_data_path(path: Path, data_dir: Path) -> str:
-    path = Path(path).resolve()
-    data_dir = Path(data_dir).resolve()
-    try:
-        return f"./{path.relative_to(data_dir)}"
-    except ValueError:
-        return str(path)
-
-
@enforce_types
-def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
+def init(force: bool=False, quick: bool=False, install: bool=False) -> None:
    """Initialize a new ArchiveBox collection in the current directory"""
-
-    from archivebox.misc.checks import check_not_inside_source_dir
-
-    check_not_inside_source_dir()
-
-    from archivebox.config import CONSTANTS, VERSION
-    from archivebox.config.common import get_config
+    
+    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
+    from archivebox.config.common import SERVER_CONFIG
    from archivebox.config.collection import write_config_file
-    from archivebox.misc.db import apply_migrations, database_exists, ensure_database_ready
-    from archivebox.misc.checks import check_migrations
+    from archivebox.misc.legacy import parse_json_main_index, parse_json_links_details, SnapshotDict
+    from archivebox.misc.db import apply_migrations
+    
+    # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
+    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
+    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)

-    config = get_config()
-
-    is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
-    existing_index = database_exists()
+    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
+    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
    if is_empty and not existing_index:
-        print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
-        print("[green]----------------------------------------------------------------------[/green]")
+        print(f'[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]')
+        print('[green]----------------------------------------------------------------------[/green]')
    elif existing_index:
        # TODO: properly detect and print the existing version in current index as well
-        print(f"[green][*] Verifying and updating existing ArchiveBox collection to v{VERSION}...[/green]")
-        print("[green]----------------------------------------------------------------------[/green]")
+        print(f'[green][*] Verifying and updating existing ArchiveBox collection to v{VERSION}...[/green]')
+        print('[green]----------------------------------------------------------------------[/green]')
    else:
        if force:
-            print("[red][!] This folder appears to already have files in it, but no index.sqlite3 is present.[/red]")
-            print("[red]    Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).[/red]")
+            print('[red][!] This folder appears to already have files in it, but no index.sqlite3 is present.[/red]')
+            print('[red]    Because --force was passed, ArchiveBox will initialize anyway (which may overwrite existing files).[/red]')
        else:
            print(
-                "[red][X] This folder appears to already have files in it, but no index.sqlite3 present.[/red]\n\n"
+                ("[red][X] This folder appears to already have files in it, but no index.sqlite3 present.[/red]\n\n"
                "    You must run init in a completely empty directory, or an existing data folder.\n\n"
                "    [violet]Hint:[/violet] To import an existing data folder make sure to cd into the folder first, \n"
                "    then run and run 'archivebox init' to pick up where you left off.\n\n"
-                "    (Always make sure your data folder is backed up first before updating ArchiveBox)",
+                "    (Always make sure your data folder is backed up first before updating ArchiveBox)"
+                )
            )
            raise SystemExit(2)

    if existing_index:
-        print("\n[green][*] Verifying archive folder structure...[/green]")
+        print('\n[green][*] Verifying archive folder structure...[/green]')
    else:
-        print("\n[green][+] Building archive folder structure...[/green]")
-
-    archive_path = _display_data_path(CONSTANTS.ARCHIVE_DIR, CONSTANTS.DATA_DIR)
-    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, CONSTANTS.DATA_DIR)
-    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, CONSTANTS.DATA_DIR)
-    print(f"    + {archive_path}, {sources_path}, {logs_path}...")
+        print('\n[green][+] Building archive folder structure...[/green]')
+    
+    print(f'    + ./{CONSTANTS.ARCHIVE_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.SOURCES_DIR.relative_to(DATA_DIR)}, ./{CONSTANTS.LOGS_DIR.relative_to(DATA_DIR)}...')
    Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
-    CONSTANTS.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
-    CONSTANTS.USERS_DIR.mkdir(parents=True, exist_ok=True)
+    Path(CONSTANTS.ARCHIVE_DIR).mkdir(exist_ok=True)
    Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
-    for path in (Path(CONSTANTS.SOURCES_DIR), CONSTANTS.ARCHIVE_DIR, CONSTANTS.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
-        path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)
-
-    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, CONSTANTS.DATA_DIR)}...")
-
+    
+    print(f'    + ./{CONSTANTS.CONFIG_FILE.relative_to(DATA_DIR)}...')
+    
    # create the .archivebox_id file with a unique ID for this collection
    from archivebox.config.paths import _get_collection_id
-
-    _get_collection_id(CONSTANTS.DATA_DIR, force_create=True)
-
+    _get_collection_id(DATA_DIR, force_create=True)
+    
    # create the ArchiveBox.conf file
-    write_config_file({"SECRET_KEY": config.SECRET_KEY})
+    write_config_file({'SECRET_KEY': SERVER_CONFIG.SECRET_KEY})

-    if existing_index:
-        print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]")
+
+    if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
+        print('\n[green][*] Verifying main SQL index and running any migrations needed...[/green]')
    else:
-        print("\n[green][+] Building main SQL index and running initial migrations...[/green]")
-
-    # For postgres, make sure the server is reachable and create the database
-    # if it doesn't exist yet (sqlite creates its file automatically).
-    ensure_database_ready()
-
+        print('\n[green][+] Building main SQL index and running initial migrations...[/green]')
+    
    from archivebox.config.django import setup_django
-
    setup_django()
-    previous_wants_init = os.environ.get("ARCHIVEBOX_WANTS_INIT")
-    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
-    try:
-        check_migrations(blocking=True, auto_apply=False)
+    
+    for migration_line in apply_migrations(DATA_DIR):
+        sys.stdout.write(f'    {migration_line}\n')

-        for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
-            sys.stdout.write(f"    {migration_line}\n")
-    finally:
-        if previous_wants_init is None:
-            os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
-        else:
-            os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
-
-    from archivebox.machine.models import Machine
-
-    # Registration requires the final schema and reconciles legacy file config.
-    Machine.current()
-
-    from archivebox.personas.models import Persona
-
-    # Initialize here, not when /add/ is rendered: a GET must not create profiles.
-    Persona.get_or_create_default().ensure_dirs()
-
-    from archivebox.misc.db import database_display_location, is_postgres
-
-    assert database_exists()
+    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
    print()
-    if is_postgres():
-        print(f"    √ {database_display_location()}")
-    else:
-        print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
+    print(f'    √ ./{CONSTANTS.DATABASE_FILE.relative_to(DATA_DIR)}')
+    
+    # from django.contrib.auth.models import User
+    # if SHELL_CONFIG.IS_TTY and not User.objects.filter(is_superuser=True).exclude(username='system').exists():
+    #     print('{green}[+] Creating admin user account...{reset}'.format(**SHELL_CONFIG.ANSI))
+    #     call_command("createsuperuser", interactive=True)

    print()
-    print("[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]")
+    print('[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]')

    from archivebox.core.models import Snapshot

-    snapshot_count = 0
+    all_links = Snapshot.objects.none()
+    pending_links: dict[str, SnapshotDict] = {}

    if existing_index:
-        snapshot_count = Snapshot.objects.count()
-        print(f"    √ Loaded {snapshot_count} links from existing main index.")
+        all_links = Snapshot.objects.all()
+        print(f'    √ Loaded {all_links.count()} links from existing main index.')

-    print("    > Skipping orphan snapshot import during init.")
-    print()
-    print("    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
-    print("        archivebox update --rescan --migrate-only")
+    if quick:
+        print('    > Skipping orphan snapshot import (quick mode)')
+    else:
+        try:
+            # Import orphaned links from legacy JSON indexes
+            orphaned_json_links = {
+                link_dict['url']: link_dict
+                for link_dict in parse_json_main_index(DATA_DIR)
+                if not all_links.filter(url=link_dict['url']).exists()
+            }
+            if orphaned_json_links:
+                pending_links.update(orphaned_json_links)
+                print(f'    [yellow]√ Added {len(orphaned_json_links)} orphaned links from existing JSON index...[/yellow]')

-    print("\n[green]----------------------------------------------------------------------[/green]")
+            orphaned_data_dir_links = {
+                link_dict['url']: link_dict
+                for link_dict in parse_json_links_details(DATA_DIR)
+                if not all_links.filter(url=link_dict['url']).exists()
+            }
+            if orphaned_data_dir_links:
+                pending_links.update(orphaned_data_dir_links)
+                print(f'    [yellow]√ Added {len(orphaned_data_dir_links)} orphaned links from existing archive directories.[/yellow]')
+
+            if pending_links:
+                for link_dict in pending_links.values():
+                    Snapshot.from_jsonl(link_dict)
+
+            # Hint for orphaned snapshot directories
+            print()
+            print('    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:')
+            print('        archivebox update')
+
+        except (KeyboardInterrupt, SystemExit):
+            print(file=sys.stderr)
+            print('[yellow]:stop_sign: Stopped checking archive directories due to Ctrl-C/SIGTERM[/yellow]', file=sys.stderr)
+            print('    Your archive data is safe, but you should re-run `archivebox init` to finish the process later.', file=sys.stderr)
+            print(file=sys.stderr)
+            print('    [violet]Hint:[/violet] In the future you can run a quick init without checking dirs like so:', file=sys.stderr)
+            print('        archivebox init --quick', file=sys.stderr)
+            raise SystemExit(1)
+
+    print('\n[green]----------------------------------------------------------------------[/green]')

    from django.contrib.auth.models import User

-    config = get_config()
-    if (config.ADMIN_USERNAME and config.ADMIN_PASSWORD) and not User.objects.filter(
-        username=config.ADMIN_USERNAME,
-    ).exists():
-        print("[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]")
-        User.objects.create_superuser(username=config.ADMIN_USERNAME, password=config.ADMIN_PASSWORD)
+    if (SERVER_CONFIG.ADMIN_USERNAME and SERVER_CONFIG.ADMIN_PASSWORD) and not User.objects.filter(username=SERVER_CONFIG.ADMIN_USERNAME).exists():
+        print('[green][+] Found ADMIN_USERNAME and ADMIN_PASSWORD configuration options, creating new admin user.[/green]')
+        User.objects.create_superuser(username=SERVER_CONFIG.ADMIN_USERNAME, password=SERVER_CONFIG.ADMIN_PASSWORD)

    if existing_index:
-        print("[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]")
+        print('[green][√] Done. Verified and updated the existing ArchiveBox collection.[/green]')
    else:
-        print(f"[green][√] Done. A new ArchiveBox collection was initialized ({snapshot_count} links).[/green]")
+        print(f'[green][√] Done. A new ArchiveBox collection was initialized ({len(all_links) + len(pending_links)} links).[/green]')

+    
    CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
    CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
-
-    from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir
-
-    config = get_config()
-    create_and_chown_dir(config.TMP_DIR)
-    create_and_chown_dir(config.ABXPKG_LIB_DIR)
-    create_and_chown_dir(config.ABXPKG_LIB_DIR / "bin")
-
-    working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
-    if working_tmp_dir:
-        working_tmp_dir.mkdir(parents=True, exist_ok=True)
-
-    working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
-    if working_lib_dir:
-        working_lib_dir.mkdir(parents=True, exist_ok=True)
-
+    CONSTANTS.DEFAULT_LIB_DIR.mkdir(parents=True, exist_ok=True)
+    
+    from archivebox.config.common import STORAGE_CONFIG
+    STORAGE_CONFIG.TMP_DIR.mkdir(parents=True, exist_ok=True)
+    STORAGE_CONFIG.LIB_DIR.mkdir(parents=True, exist_ok=True)
+    
    if install:
        from archivebox.cli.archivebox_install import install as install_method
-
        install_method()

-    if Snapshot.objects.count() < 25:  # hide the hints for experienced users
-        from archivebox.core.routes_util import build_admin_url
-
-        admin_url = build_admin_url("/admin/", config=config)
+    if Snapshot.objects.count() < 25:     # hide the hints for experienced users
        print()
-        print("    [violet]Hint:[/violet] To finish web setup, run:")
-        print(
-            f"        archivebox server  # then visit [deep_sky_blue4][link={admin_url}]{admin_url}[/link][/deep_sky_blue4]",
-        )
+        print('    [violet]Hint:[/violet] To view your archive index, run:')
+        print('        archivebox server  # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]')
        print()
-        print("    To add new links, you can run:")
+        print('    To add new links, you can run:')
        print("        archivebox add < ~/some/path/to/list_of_links.txt")
        print()
-        print("    For more usage and examples, run:")
-        print("        archivebox help")
+        print('    For more usage and examples, run:')
+        print('        archivebox help')
+


@click.command()
-@click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
-@click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
-@click.option("--install", "--setup", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
+@click.option('--force', '-f', is_flag=True, help='Ignore unrecognized files in current directory and initialize anyway')
+@click.option('--quick', '-q', is_flag=True, help='Run any updates or migrations without rechecking all snapshot dirs')
+@click.option('--install', '-s', is_flag=True, help='Automatically install dependencies and extras used for archiving')
@docstring(init.__doc__)
 def main(**kwargs) -> None:
    init(**kwargs)


-if __name__ == "__main__":
+if __name__ == '__main__':
    main()
@@ -1,11 +1,10 @@
 #!/usr/bin/env python3

-__package__ = "archivebox.cli"
+__package__ = 'archivebox.cli'

 import os
+import sys
 import shutil
-import re
-from pathlib import Path

 import rich_click as click
 from rich import print
@@ -13,222 +12,108 @@ from rich import print
 from archivebox.misc.util import docstring, enforce_types


-def _resolve_install_targets(
-    requested_names: tuple[str, ...],
-) -> tuple[list[str], list[str]]:
-    """Resolve plugin names and declared binary aliases, leaving unknown tokens raw."""
-    from archivebox.plugins.discovery import get_plugin_catalog
-
-    catalog = get_plugin_catalog()
-    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in catalog}
-    plugin_names_by_binary_alias: dict[str, set[str]] = {}
-    for plugin_name, plugin in catalog.items():
-        for required_binary in plugin.config.required_binaries:
-            aliases = {required_binary.name.lower()}
-            template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", required_binary.name)
-            if template_match:
-                config_key = template_match.group(1)
-                aliases.add(config_key.lower())
-                aliases.add(config_key.removesuffix("_BINARY").lower())
-                default = plugin.config.properties.get(config_key, {}).get("default")
-                if default:
-                    aliases.add(str(default).lower())
-            for alias in aliases:
-                plugin_names_by_binary_alias.setdefault(alias, set()).add(plugin_name)
-
-    requested_plugins: list[str] = []
-    raw_binary_names: list[str] = []
-    for name in requested_names:
-        normalized_name = name.lower()
-        if normalized_name in plugin_names_by_lower:
-            requested_plugins.append(plugin_names_by_lower[normalized_name])
-        elif normalized_name in plugin_names_by_binary_alias:
-            requested_plugins.extend(sorted(plugin_names_by_binary_alias[normalized_name]))
-        else:
-            raw_binary_names.append(name)
-
-    selected_plugins = catalog.select(requested_plugins) if requested_plugins else catalog.select([])
-    selected_plugin_names = {name.lower() for name in selected_plugins}
-    raw_binary_names = [name for name in raw_binary_names if name.lower() not in selected_plugin_names]
-    return sorted(selected_plugins), sorted(set(raw_binary_names))
-
-
-def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> None:
-    """Install user-requested standalone binaries through the Binary lifecycle."""
-    from django.utils import timezone
-
-    from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-
-    machine = Machine.current()
-    for requested_name in binary_names:
-        binary_name = _canonical_binary_name(requested_name)
-        if not binary_name:
-            raise click.UsageError(f"Invalid binary name: {requested_name}")
-        binary = Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").first()
-        if binary is None:
-            binary = Binary.objects.create(
-                machine=machine,
-                name=binary_name,
-                binproviders=binproviders,
-                overrides={},
-                status=Binary.StatusChoices.QUEUED,
-            )
-        elif not binary.is_valid or binary.binproviders != binproviders:
-            binary.binproviders = binproviders
-            binary.overrides = {}
-            binary.status = Binary.StatusChoices.QUEUED
-            binary.retry_at = timezone.now()
-            binary.save(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
-        if binary.status == Binary.StatusChoices.INSTALLED:
-            continue
-        # The synchronous CLI needs the lifecycle's verified installation result,
-        # not the scheduler's "claimed work" result. Reuse its failure/requeue and
-        # health-stat handling so a failed install cannot exit successfully.
-        if not binary.install_claimed(lock_seconds=60):
-            raise click.ClickException(f"Binary {binary_name} is already being installed")
-
-
-def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:
-    """Expose the shared abxpkg lib dir at DATA_DIR/lib after install."""
-    link_path = data_dir / "lib"
-    target_config_path = abxpkg_lib_dir.expanduser().absolute()
-    target_path = abxpkg_lib_dir.expanduser().resolve()
-    from archivebox.config.paths import get_machine_type
-
-    scoped_data_lib_path = (link_path / get_machine_type()).absolute()
-
-    if target_config_path == scoped_data_lib_path:
-        if link_path.is_symlink() or link_path.is_file():
-            link_path.unlink()
-        target_config_path.mkdir(parents=True, exist_ok=True)
-        return None
-
-    if link_path.resolve(strict=False) == target_path:
-        return None
-
-    target_path.mkdir(parents=True, exist_ok=True)
-
-    if link_path.is_symlink():
-        if link_path.resolve(strict=False) == target_path:
-            return link_path
-        link_path.unlink()
-    elif link_path.exists():
-        shutil.rmtree(link_path)
-
-    relative_target = os.path.relpath(target_path, start=link_path.parent)
-    link_path.symlink_to(relative_target, target_is_directory=True)
-    return link_path
-
-
@enforce_types
-def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
-    """Detect and install ArchiveBox dependencies by running the abx-dl install flow
-
-    Examples:
-        archivebox install                              # Install all dependencies
-        archivebox install wget curl                    # Install only wget and curl
-        archivebox install --binproviders=pip yt-dlp    # Install yt-dlp using only pip
-        archivebox install --binproviders=brew,apt      # Install all deps using only brew or apt
-    """
+def install(dry_run: bool=False) -> None:
+    """Detect and install ArchiveBox dependencies by running a dependency-check crawl"""

    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
-    from archivebox.config import CONSTANTS
+    from archivebox.config.paths import ARCHIVE_DIR
    from archivebox.misc.logging import stderr
    from archivebox.cli.archivebox_init import init

-    archive_dir = CONSTANTS.ARCHIVE_DIR
-
-    if dry_run:
-        print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")
-        return
-
-    if not (os.access(archive_dir, os.R_OK) and archive_dir.is_dir()):
+    if not (os.access(ARCHIVE_DIR, os.R_OK) and ARCHIVE_DIR.is_dir()):
        init()  # must init full index because we need a db to store Binary entries in

-    # Show what we're installing
-    if binaries:
-        print(f"\n[green][+] Installing specific binaries: {', '.join(binaries)}[/green]")
-    else:
-        print("\n[green][+] Detecting and installing all ArchiveBox dependencies...[/green]")
-
-    if binproviders != "*":
-        print(f"[green][+] Using providers: {binproviders}[/green]")
+    print('\n[green][+] Detecting ArchiveBox dependencies...[/green]')

    if IS_ROOT:
        EUID = os.geteuid()
        print()
-        print(f"[yellow]:warning:  Running as UID=[blue]{EUID}[/blue].[/yellow]")
-        print(f"    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].")
+        print(f'[yellow]:warning:  Running as UID=[blue]{EUID}[/blue].[/yellow]')
+        print(f'    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].')
        print()

+    if dry_run:
+        print('[dim]Dry run - would create a crawl to detect dependencies[/dim]')
+        return
+
    # Set up Django
    from archivebox.config.django import setup_django
-
    setup_django()

-    plugin_names, raw_binary_names = _resolve_install_targets(binaries) if binaries else ([], [])
-    install_plugin_names = list(plugin_names)
-    if binproviders != "*":
-        install_plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
+    from django.utils import timezone
+    from archivebox.crawls.models import Crawl
+    from archivebox.base_models.models import get_or_create_system_user_pk

-    if install_plugin_names or not binaries:
-        if not binaries:
-            from archivebox.workers.supervisord_util import require_dependency_install_memory
+    # Create a crawl for dependency detection
+    # Using a minimal crawl that will trigger on_Crawl hooks
+    created_by_id = get_or_create_system_user_pk()

-            require_dependency_install_memory()
-        print("[+] Running plugin installer via abx-dl bus...")
-        print()
+    crawl, created = Crawl.objects.get_or_create(
+        urls='archivebox://install',
+        defaults={
+            'label': 'Dependency detection',
+            'created_by_id': created_by_id,
+            'max_depth': 0,
+            'status': 'queued',
+        }
+    )

-        from archivebox.services.runner import run_install
+    # If crawl already existed, reset it to queued state so it can be processed again
+    if not created:
+        crawl.status = 'queued'
+        crawl.retry_at = timezone.now()
+        crawl.save()

-        run_install(plugin_names=install_plugin_names or None)
+    print(f'[+] Created dependency detection crawl: {crawl.id}')
+    print(f'[+] Crawl status: {crawl.status}, retry_at: {crawl.retry_at}')

-    if raw_binary_names:
-        print(f"[+] Running direct binary installer via ArchiveBox binary lifecycle: {', '.join(raw_binary_names)}")
-        print()
-        _install_raw_binary_names(raw_binary_names, binproviders)
+    # Verify the crawl is in the queue
+    from archivebox.crawls.models import Crawl as CrawlModel
+    queued_crawls = CrawlModel.objects.filter(
+        retry_at__lte=timezone.now()
+    ).exclude(
+        status__in=CrawlModel.FINAL_STATES
+    )
+    print(f'[+] Crawls in queue: {queued_crawls.count()}')
+    if queued_crawls.exists():
+        for c in queued_crawls:
+            print(f'    - Crawl {c.id}: status={c.status}, retry_at={c.retry_at}')

-    from archivebox.config.common import get_config
+    print('[+] Running crawl to detect binaries via on_Crawl hooks...')
+    print()

-    config = get_config(include_machine=False)
-    ensure_data_dir_lib_symlink(CONSTANTS.DATA_DIR, config.ABXPKG_LIB_DIR)
+    # Run the crawl synchronously (this triggers on_Crawl hooks)
+    from archivebox.workers.orchestrator import Orchestrator
+    orchestrator = Orchestrator(exit_on_idle=True)
+    orchestrator.runloop()

    print()

    # Check for superuser
    from django.contrib.auth import get_user_model
-
    User = get_user_model()

-    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
-        from archivebox.core.routes_util import build_admin_url
-
-        stderr("\n[+] Open the Admin UI to create the first admin and finish web setup:", color="green")
-        stderr(f"    {build_admin_url('/admin/', config=get_config(resolve_plugins=False))}")
+    if not User.objects.filter(is_superuser=True).exclude(username='system').exists():
+        stderr('\n[+] Don\'t forget to create a new admin user for the Web UI...', color='green')
+        stderr('    archivebox manage createsuperuser')

    print()

-    # Show version to display full status including installed binaries
-    # Django is already loaded, so just import and call the function directly
-    from archivebox.cli.archivebox_version import version as show_version
-
-    show_version(quiet=False, binaries=binaries)
+    # Run version to show full status
+    archivebox_path = shutil.which('archivebox') or sys.executable
+    if 'python' in archivebox_path:
+        os.system(f'{sys.executable} -m archivebox version')
+    else:
+        os.system(f'{archivebox_path} version')


@click.command()
-@click.argument("binaries", nargs=-1, type=str, required=False)
-@click.option(
-    "--binproviders",
-    "-p",
-    default="*",
-    help="Comma-separated list of providers to use (pip,npm,brew,apt,env,custom) or * for all",
-    show_default=True,
-)
-@click.option("--dry-run", "-d", is_flag=True, help="Show what would happen without actually running", default=False)
+@click.option('--dry-run', '-d', is_flag=True, help='Show what would happen without actually running', default=False)
@docstring(install.__doc__)
 def main(**kwargs) -> None:
    install(**kwargs)
+    

-
-if __name__ == "__main__":
+if __name__ == '__main__':
    main()