Fix runner and persona CI races

.dockerignore

@@ -13,14 +13,10 @@ __pycache__
 .pytest_cache/
 .ruff_cache/
 .uv-cache/
-.cache/
-.cache*/
 .github/
 .pdm-build/
-.pdm-build/**
 .pdm-python
 .eggs/
-*.egg-info/
 .git/
 !.git/
 .git/*
@@ -46,7 +42,6 @@ chromeprofile/
 chrome_profile/
 lib/
 out/
-tests/out/
 users/
 archive/
 crawls/
@@ -68,22 +63,11 @@ dist/
 brew_dist/
 deb_dist/
 pip_dist/
-cache/
-caches/
 assets/
 docker/
 website/
 typings/
 
-# The runtime COPY needs ArchiveBox's package and bin/ entrypoint, not the
-# development UI screenshots or test trees. PNGs are already compressed, so
-# including them creates a large image layer that later cleanup cannot remove.
-publicsite/
-old/
-test/
-tests/
-archivebox/tests/
-
 tmp/
 .tmp/
 data/

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@
 - https://github.com/ArchiveBox/ArchiveBox/issues
 - https://github.com/ArchiveBox/ArchiveBox/pulls
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
 
 ### Development Setup
 

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -30,8 +30,8 @@ body:
       description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
       render: markdown
       placeholder: |
-        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 5797:5797 archivebox/archivebox` in iTerm2
-        2. Went to the https://127.0.0.1:5797/add/ page in Google Chrome
+        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
+        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
         3. Typed 'https://example.com/xyz' into the 'Add URL' input field
         4. Clicked the 'Add+' button
         5. Got a 500 error and saw the errors below in terminal
@@ -69,7 +69,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:
@@ -138,7 +138,7 @@ body:
             archivebox:
                 image: archivebox/archivebox:latest
                 ports:
-                    - 5797:5797
+                    - 8000:8000
                 volumes:
                     - ./data:/data
                 environment:
@@ -191,6 +191,7 @@ body:
         In the meantime please consider:  
           
           - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
           - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
           - 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
           - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)

.github/ISSUE_TEMPLATE/2-feature_request.yml

@@ -71,7 +71,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:
@@ -109,6 +109,9 @@ body:
           required: false
         - label: "I've had a lot of difficulty getting ArchiveBox set up"
           required: false
+        - label: "I would pay $10/mo for a hosted version of ArchiveBox if it had this feature"
+          required: false
+
   - type: markdown
     attributes:
       value: |
@@ -120,5 +123,6 @@ body:
         In the meantime please consider:  
           
           - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 📊 [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
           - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
           - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)

.github/ISSUE_TEMPLATE/3-documentation_change.yml

@@ -47,5 +47,6 @@ body:
         In the meantime please consider:  
           
           - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
           - 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
           - ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)

.github/ISSUE_TEMPLATE/config.yml

@@ -6,3 +6,6 @@ contact_links:
   - name: 💬 Chat with the dev team & community on Zulip
     url: https://zulip.archivebox.io
     about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
+  - name: 💁‍♂️ Hire us for professional support with fast response times
+    url: https://docs.monadical.com/s/archivebox-consulting-services
+    about: "We provide hosting, development, and support, including on-prem/cloud w/ SSO & storage, CAPTCHA-solving, proxies, etc."

.github/SECURITY.md

@@ -2,15 +2,18 @@
 
 ---
 
-## Security Documentation
+## Security Information
 
-Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
+Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
+
+https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+
+Also see this section of the README about important caveats when running ArchiveBox:
+
+https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
+
+You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
 
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
-- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
-- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
 - https://github.com/ArchiveBox/ArchiveBox#archive-layout
 - https://github.com/ArchiveBox/ArchiveBox#archivebox-development
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
@@ -20,49 +23,12 @@ Please start by reading these links to understand our existing security model, k
 
 ## Reporting a Vulnerability
 
-~We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.~ We no longer offer CVEs or accept private reporting, 95% of the reports we received were low quality AI slop, so now we just ask for normal Github Issues or PRs.
+We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.
 
-You can contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp) if you have a truly urgent security report, but we do not issue public CVEs.
+1. Go to the Security tab on our Github repo: https://github.com/ArchiveBox/ArchiveBox/security
 
----
+2. Click the ["Report a Vulnerability"](https://github.com/ArchiveBox/ArchiveBox/security/advisories/new) button
 
-## CVEs and Report Credits
+3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
 
-> [!IMPORTANT]
-> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**  
-> Reporters **will still receive credit in release notes** however.
-
-> [!TIP]
-> ~Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.~
-
-Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
-
----
-
-## Threat Model
-
-### All admin users are assumed to be trusted
-
-ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
-
-- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
-- non-logged in anonymous guest users
-
-Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
-
-### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
-
-Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
-
-Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
-
-### XSS, CSRF, CORS, CSP
-
-`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
-
-[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
-
-### SSRF / URL Validation
-
-Note archivebox is **designed to be able to archive any URL the server can reach**, including internal intranet URLs, private IPs, and localhost. Users are expected to configure `URL_ALLOWLIST`/`URL_DENYLIST` or proper firewall/ACL-level blocking at the network level if they need to limit what the server should see.
-It does a basic check to try and avoid recursively archiving snapshots hosted by the archivebox server itself, but it's not a strict security check, it's just to prevent wasted CPU/disk from recursively archiving snapshots in a browser that has the ArchiveBox Browser Extension turned on.
+You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).

.github/STAGING.md

@@ -1,42 +0,0 @@
-# Stable releases require live dev acceptance
-
-The order is **dev CI → dev image publication → automatic Cabbage deployment →
-real capture and browser acceptance → stable publication**. Never
-substitute a published PyPI wheel, healthy container, or local build for this
-acceptance. The release workflow blocks *before* a stable upload unless Cabbage
-has successful evidence from `staging-acceptance.yml` matching all tracked
-source and dependency pins. Only ArchiveBox's own version fields may change
-between dev and stable.
-
-The staging workflow observes the existing image watcher; it cannot deploy or
-restart the application. On Cabbage, install `bin/staging-acceptance-host.sh`
-as root-owned mode 0755 `/usr/local/sbin/archivebox-staging-acceptance`. Its hash
-is checked against the candidate. A dedicated SSH key must use
-`restrict,command="/usr/local/sbin/archivebox-staging-acceptance"` in authorized
-keys. It accepts only `inspect <source-sha>` and `capture <source-sha>`; it cannot
-run caller-provided commands or archive arbitrary URLs.
-
-The `staging` GitHub environment is restricted to dev/main branches. It contains
-`STAGING_SSH_KEY` and `STAGING_KNOWN_HOSTS`, with independently verified host keys.
-Only trusted release events use the private ugNAS runner route. Never expose
-these credentials to PR jobs or copy an operator's existing root key.
-
-Acceptance creates one tagged background crawl per source through the normal
-add/runner path, enabling every snapshot extractor, including TLSNotary. It
-checks installed pins, completed outputs and their files, unexpected/missing
-plugin results, host OOM kills, and real rendered snapshot pages. For this
-release, the explicitly accepted TLSNotary failures and exact missing-Claude-auth
-diagnostics are retained in the report's `limitations` list; they do not disable
-those plugins or excuse missing/truncated successful outputs. Remove the
-TLSNotary exception when its follow-up fix ships. Logs, result inventories and
-screenshots are retained as Actions artifacts. Other failures remain
-failures; do not publish a success deployment record to bypass them. No code
-mounts are allowed. The Cabbage job must pass before its record authorizes a
-stable upload. Queued capture progress is polled; failed tests are not retried.
-
-Browser acceptance follows the default WACZ viewer into the captured document,
-matching its replay base and original URL and checking that its body is visible
-and contains substantial text. An attached viewer iframe can still be
-`about:blank`, so its presence alone cannot prove replay works. Extracted display
-titles and the captured document's `<title>` may legitimately differ; neither
-title equality nor output thumbnails substitute for checking the actual page.

.github/actionlint.yaml

@@ -1,3 +0,0 @@
-self-hosted-runner:
-  labels:
-    - ugnas-linux

.github/configs/ci-linux-build.json

@@ -1,81 +0,0 @@
-{
-  "properties": {
-    "CC_BINARY": {
-      "default": "cc"
-    },
-    "LDAPSEARCH_BINARY": {
-      "default": "ldapsearch"
-    },
-    "INITDB_BINARY": {
-      "default": "initdb"
-    },
-    "PG_CTL_BINARY": {
-      "default": "pg_ctl"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{CC_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "build-essential"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{LDAPSEARCH_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapsearch",
-            "-VV"
-          ]
-        },
-        "apt": {
-          "install_args": [
-            "ldap-utils",
-            "python3-dev",
-            "python3-setuptools",
-            "libssl-dev",
-            "libldap2-dev",
-            "libsasl2-dev",
-            "zlib1g-dev",
-            "libatomic1"
-          ],
-          "version": [
-            "ldapsearch",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ],
-  "postgres_binaries": [
-    {
-      "name": "{INITDB_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "postgresql"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{PG_CTL_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "postgresql"
-          ]
-        }
-      }
-    }
-  ]
-}

.github/configs/ci-macos-brew.json

@@ -1,13 +0,0 @@
-{
-  "properties": {
-    "BREW_BINARY": {
-      "default": "brew"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{BREW_BINARY}",
-      "binproviders": "env"
-    }
-  ]
-}

.github/configs/ci-macos-build.json

@@ -1,44 +0,0 @@
-{
-  "properties": {
-    "CLANG_BINARY": {
-      "default": "clang"
-    },
-    "LDAPVC_BINARY": {
-      "default": "ldapvc"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{CLANG_BINARY}",
-      "binproviders": "env,brew",
-      "overrides": {
-        "brew": {
-          "install_args": [
-            "llvm"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{LDAPVC_BINARY}",
-      "binproviders": "env,brew",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        },
-        "brew": {
-          "install_args": [
-            "openldap"
-          ],
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ]
-}

.github/configs/ci-macos-openldap-env.json

@@ -1,21 +0,0 @@
-{
-  "properties": {
-    "LDAPVC_BINARY": {
-      "default": "ldapvc"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{LDAPVC_BINARY}",
-      "binproviders": "env",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ]
-}

.github/configs/ci-tooling.json

@@ -1,349 +0,0 @@
-{
-  "properties": {
-    "CURL_BINARY": {
-      "default": "curl"
-    },
-    "BASH_BINARY": {
-      "default": "bash"
-    },
-    "CHMOD_BINARY": {
-      "default": "chmod"
-    },
-    "DOCKER_BINARY": {
-      "default": "docker"
-    },
-    "GIT_BINARY": {
-      "default": "git"
-    },
-    "GREP_BINARY": {
-      "default": "grep"
-    },
-    "GH_BINARY": {
-      "default": "gh"
-    },
-    "EXPORTFS_BINARY": {
-      "default": "exportfs"
-    },
-    "JQ_BINARY": {
-      "default": "jq"
-    },
-    "MKDIR_BINARY": {
-      "default": "mkdir"
-    },
-    "MOUNT_CIFS_BINARY": {
-      "default": "mount.cifs"
-    },
-    "MOUNT_NFS_BINARY": {
-      "default": "mount.nfs"
-    },
-    "NODE_BINARY": {
-      "default": "node"
-    },
-    "PS_BINARY": {
-      "default": "ps"
-    },
-    "PYTHON_BINARY": {
-      "default": "python"
-    },
-    "PV_BINARY": {
-      "default": "pv"
-    },
-    "SSH_BINARY": {
-      "default": "ssh"
-    },
-    "SED_BINARY": {
-      "default": "sed"
-    },
-    "SMBD_BINARY": {
-      "default": "smbd"
-    },
-    "SUDO_BINARY": {
-      "default": "sudo"
-    },
-    "SYSTEMCTL_BINARY": {
-      "default": "systemctl"
-    },
-    "TAR_BINARY": {
-      "default": "tar"
-    },
-    "TREE_BINARY": {
-      "default": "tree"
-    },
-    "TR_BINARY": {
-      "default": "tr"
-    },
-    "RM_BINARY": {
-      "default": "rm"
-    },
-    "UMOUNT_BINARY": {
-      "default": "umount"
-    },
-    "UV_BINARY": {
-      "default": "uv"
-    },
-    "UNAME_BINARY": {
-      "default": "uname"
-    }
-  },
-  "ci_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{JQ_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{UV_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "bsd_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,pyinfra"
-    }
-  ],
-  "freebsd_binaries": [
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env,pyinfra"
-    }
-  ],
-  "docker_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UNAME_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "deploy_binaries": [
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SSH_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "docker_debug_binaries": [
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{PV_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{TAR_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{TREE_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "docker_validation_binaries": [
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GREP_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{MKDIR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{RM_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SED_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SSH_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{TR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UNAME_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "mount_validation_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{CHMOD_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{EXPORTFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "nfs-kernel-server"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{MKDIR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{MOUNT_CIFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "cifs-utils"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{MOUNT_NFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "nfs-common"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{SMBD_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "samba"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SYSTEMCTL_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UMOUNT_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "process_binaries": [
-    {
-      "name": "{PS_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "docs_binaries": [
-    {
-      "name": "{CURL_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "git_binaries": [
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "github_binaries": [
-    {
-      "name": "{GH_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "node_binaries": [
-    {
-      "name": "{NODE_BINARY}",
-      "binproviders": "env,npm",
-      "min_version": "22.23.1",
-      "overrides": {
-        "npm": {
-          "install_args": [
-            "node@22.23.1"
-          ],
-          "postinstall_scripts": true
-        }
-      }
-    }
-  ],
-  "release_binaries": [
-    {
-      "name": "{UV_BINARY}",
-      "binproviders": "env,pip,brew"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{JQ_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{CURL_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ]
-}

.github/dependabot.yml

@@ -15,7 +15,7 @@ updates:
         patterns:
           - "*"
   - package-ecosystem: "npm"
-    directory: "/etc"
+    directory: "/"
     target-branch: "dev"
     schedule:
       interval: "monthly"

.github/lychee.toml

@@ -1,38 +0,0 @@
-# Documentation destinations only: do not crawl archived URLs, app routes, or image assets.
-# Manual run from the repository root with Lychee 0.24.2:
-# git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > /tmp/archivebox-link-inputs.txt
-# lychee --config .github/lychee.toml --root-dir "$PWD" --files-from /tmp/archivebox-link-inputs.txt
-# Explicit tracked inputs include README symlinks and let errors in one file leave others checkable.
-include_fragments = "anchor-only"
-include_verbatim = true
-max_retries = 0
-max_concurrency = 4
-timeout = 20
-no_progress = true
-hidden = true
-extensions = ["md", "html", "py", "rst", "yml", "yaml", "sh", "toml"]
-exclude_path = ['(^|/)(tests|data|apidocs|node_modules|\.git)(/|$)']
-# Includes override this exclude; this is a documentation-link allowlist.
-exclude = ['.*']
-include = [
- '^https://github\.com/ArchiveBox/ArchiveBox/blob/[^/]+/.*\.md(?:#|$)',
- '^https://raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/',
- '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki(?:/|#|$)|/?#)',
- '^https?://docs\.archivebox\.io(?:/|$)',
- '^file:.*\.md(?:#|$)',
- '^file:.*/docs/[^/.#]+(?:#|$)',
-]
-fallback_extensions = ["md"]
-remap = [
- # Lychee's Markdown slugger drops variation selectors. Check these against rendered HTML.
-
- '^file:.*/README\.md(#%EF%B8%8F-cli-usage)$ https://github.com/ArchiveBox/ArchiveBox${1}',
- '^file:.*/docs/Install\.md(#option-a-docker--docker-compose-setup-%EF%B8%8F)$ https://github.com/ArchiveBox/ArchiveBox/wiki/Install${1}',
- # The wiki accepts this case alias, but raw file paths are case-sensitive.
- '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/Setting-up-Storage$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/Setting-Up-Storage.md',
-
- # Bare missing wiki pages silently redirect to Home; raw pages return an actual 404.
- # Keep fragment URLs on the rendered wiki to catch published heading drift.
- '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/([^#/?]+)$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/${1}.md',
-]
-preprocess = { command = "bin/lychee_preprocess.py" }

.github/pages/README.md

@@ -1,124 +0,0 @@
-# archivebox.io website
-
-This repository owns its complete website. Build and deploy it without checking
-out another repository's website, fetching shared snippets, or calling a shared
-workflow. Common files are ordinary committed copies; update them deliberately.
-
-## Repository-specific files
-
-- Content: `publicsite/index.html`.
-- Site/gallery generation: `bin/generate_ui_screenshot_gallery.py`.
-- Application capture: `bin/collect_ui_screenshots.sh`.
-- Deployment branch: `dev`.
-
-## Common publication contract
-
-1. Build pull requests without deploying them.
-2. Build website pushes immediately, independently of application capture CI.
-3. Restore validated screenshots from trusted default-branch capture jobs.
-   Keep the original capture revision, timestamp, coverage and image checksums.
-4. Retain `site-screenshots` artifacts for 90 days. After expiry, restore the
-   published image set. Never replace a complete gallery with an empty one when
-   restoration fails. Apple device groups are independently validated artifacts;
-   Android's checked-in bootstrap captures retain their local provenance.
-5. Render current content and layout around those captures. Site revision and
-   capture revision are separate facts; reusing captures must not relabel them.
-6. Serialize production restore/build/deploy under one concurrency group and
-   check out the latest default-branch revision when each build starts. Keep the
-   built revision in provenance; later release-bot commits must not block deploys.
-7. Keep platform-specific build, installation, capture, and completeness checks
-   in the application's existing capture workflow. Site-only edits do not need
-   to rerun those jobs.
-
-## Common presentation contract
-
-The base header, Apps menu and icons, footer, brand styles and navigation behavior
-should remain identical copies across the seven product sites. Put this site's
-name, navigation, CTA and extra footer resources outside the common base files.
-Preserve real links, anchor IDs, no-JavaScript navigation, keyboard operation,
-mobile layout, reduced-motion preferences and existing product-specific content.
-Screenshots stay in English. Translation content, when added, lives in separate
-HTML files; no runtime translation service or cross-repository dependency.
-
-## Editing and building
-
-`base/` contains identical independent copies across the seven product sites.
-Common Python scripts share the same behavior; repository formatters may change
-line wrapping without changing their syntax trees. The base owns the shared logo, Apps menu/icons, footer directory, navigation
-behavior and styles. `site.json` and `nav.html` contain this site's name, routes,
-links and CTA. Product content and additional footer resources stay in the native
-HTML/Jinja/Liquid templates listed above. The `ARCHIVEBOX:*` comments are build
-slots; the renderer fills them with static HTML, never browser-side fetches.
-
-The existing builder calls `site.py render` after generating its content. This
-copies `base/chrome.css` and `base/chrome.js` to `site-base/`, assembles the common
-HTML, validates local resources and writes the site revision to root `build.json`.
-Screenshot manifests are left untouched. No build reads another repository.
-
-Common changes are manual copy/paste edits. Compare `base/` and `site.py` with
-another product repository when updating them; do not add a synchronization job,
-package dependency or reusable cross-repository workflow. `nav.html`, `site.json`
-and the marked workflow build section are intentionally repository-specific.
-
-Build: `uv run --no-project python .github/pages/site.py render _site --source publicsite` after restoring screenshots and refreshing the gallery.
-
-Check the built site with real Chromium (also run by Pages CI):
-
-```sh
-export ABXPKG_LIB_DIR=/tmp/archivebox-site-browser
-CHROME_SCHEMA="$(uv run python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
-uv run abxpkg env --install --deps-from="$CHROME_SCHEMA:required_binaries"
-export CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
-uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py SITE_OUTPUT --evidence /tmp/site-evidence
-```
-
-The browser comes from the normal abxpkg provider/cache path; Playwright drives
-that resolved executable instead of installing a second Chromium build.
-Use this workflow's `SITE_OUTPUT` directory. The check covers desktop/mobile
-layout, keyboard dismissal, no-JavaScript links, local resources and visible
-images, and uploads its screenshots as `site-verification` for 14 days.
-
-## Branding and future translations
-
-`base/chrome.css` owns the shared system font stacks, plum accent and CTA shape,
-including the existing `.button` and `.ab-btn` classes. Product styles own their
-layouts, feature sections and platform-specific button states. Shared system
-fonts need no external font downloads. `site.py` emits the same complete SEO/OG/
-Twitter tags everywhere while retaining each native template's title, description,
-canonical URL, locale and social artwork; edit those values with that page's copy.
-
-For translations, keep complete separate HTML files under `es/`, `fr/` and `zh/`,
-and list their output paths in `site.json` alongside the English pages. Share CSS,
-JS, social artwork and the English screenshot gallery. Translate each page's title
-and description, set its own canonical and `lang` (`zh` for Chinese), and add
-reciprocal `hreflang` links only for translations that actually exist. The renderer
-preserves these tags and computes shared asset paths for nested pages. Localized
-header/footer copy and the small language selector can be added with those pages;
-there is no language redirect, placeholder translation or translation runtime now.
-
-## Screenshot strips
-
-`marquee.py` and `base/marquee.css` / `base/marquee.js` are also identical copies.
-Product homepages choose their placement with an `ARCHIVEBOX:MARQUEE` comment.
-The renderer takes real images, captions, dimensions and links from the generated
-`screenshots/index.html`; there is no second screenshot list or browser fetch.
-Desktop/tablet/mobile galleries contribute their desktop views. Apple uses
-`MARQUEE-CLIENT` and `MARQUEE-SERVER` slots to keep its two products separate.
-Catalog sites without a slot do not load the strip assets.
-
-The strip scrolls once, pauses on hover, focus or manual interaction, and has an
-explicit Play/Pause button. Reduced-motion users start paused. Without JavaScript,
-normal image links and horizontal scrolling remain available. Screenshot capture
-manifests and provenance are never rewritten by this presentation step.
-
-Standalone one- or two-image illustrations remain in their authored source.
-Gallery strips with three or more images use the latest restored captures.
-
-## Keeping the common layout in sync
-
-The comments in `base/header.html`, `base/footer-start.html`,
-`base/footer-end.html`, `base/chrome.css`, and `base/chrome.js` list the peer
-repositories. Update those committed copies together when shared branding,
-Apps links, keyboard behavior, or footer resources change. Preserve each site's
-local navigation and content. The former monorepo `site-chrome` generator is
-retired; builds must use only this repository's files.

.github/pages/artifacts.py

@@ -1,102 +0,0 @@
-"""Common screenshot artifact transport; coverage checks belong to each app."""
-
-import json
-import subprocess
-from pathlib import PurePosixPath
-from urllib.request import urlopen
-
-
-def api(repo, path):
-    return json.loads(subprocess.check_output(["gh", "api", f"repos/{repo}/{path}"]))
-
-
-def trusted(run, repo, workflow, branch, successful=True):
-    return (
-        (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
-        and run["head_branch"] == branch
-        and run["event"] in ("push", "workflow_dispatch", "schedule")
-        and run["path"] == f".github/workflows/{workflow}"
-        and (not successful or run["conclusion"] == "success")
-    )
-
-
-def runs(repo, workflow, branch, successful=True, artifact_names=("site-screenshots",)):
-    # Ask for capture artifacts directly instead of checking every historical
-    # workflow run (most app runs never produce a complete screenshot set).
-    # Artifact creation time also includes fresh captures from old rerun jobs.
-    candidates = []
-    for name in artifact_names:
-        rows = subprocess.check_output(
-            [
-                "gh",
-                "api",
-                f"repos/{repo}/actions/artifacts?name={name}&per_page=100",
-                "--paginate",
-                "--jq",
-                ".artifacts[] | select(.expired == false) | @json",
-            ],
-            text=True,
-        )
-        candidates.extend(json.loads(row) for row in rows.splitlines())
-    seen = set()
-    for artifact in sorted(candidates, key=lambda item: item["created_at"], reverse=True):
-        source = artifact.get("workflow_run") or {}
-        run_id = source.get("id")
-        if (
-            artifact["name"] not in artifact_names
-            or not run_id
-            or run_id in seen
-            or source.get("head_branch") != branch
-            or source.get("head_repository_id") != source.get("repository_id")
-        ):
-            continue
-        seen.add(run_id)
-        run = api(repo, f"actions/runs/{run_id}")
-        if trusted(run, repo, workflow, branch, successful):
-            yield run
-
-
-def names(repo, run):
-    rows = subprocess.check_output(
-        [
-            "gh",
-            "api",
-            f"repos/{repo}/actions/runs/{run['id']}/artifacts?per_page=100",
-            "--paginate",
-            "--jq",
-            ".artifacts[] | @json",
-        ],
-        text=True,
-    )
-    return {artifact["name"] for row in rows.splitlines() if not (artifact := json.loads(row))["expired"]}
-
-
-def download(repo, run, name, destination):
-    subprocess.run(
-        [
-            "gh",
-            "run",
-            "download",
-            str(run["id"]),
-            "--repo",
-            repo,
-            "--name",
-            name,
-            "--dir",
-            str(destination),
-        ],
-        check=True,
-    )
-
-
-def relative_path(name):
-    path = PurePosixPath(name)
-    if not name or path.is_absolute() or ".." in path.parts or any(c in name for c in ":\\?#%"):
-        raise ValueError(f"Unsafe screenshot path: {name}")
-    return path
-
-
-def fetch(base, name):
-    relative_path(name)
-    with urlopen(base + name, timeout=60) as response:
-        return response.read()

.github/pages/base/chrome.css

@@ -1,156 +0,0 @@
-/*
-Shared ArchiveBox website navigation styles and behavior.
-Keep this file in sync with .github/pages/base/chrome.css in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
-*/
-/* Common brand tokens. Native templates own product layout and content. */
-:root {
-  --abx-font-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
-  --abx-font-mono: ui-monospace, "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace;
-  --abx-accent: #9b2854;
-  --abx-accent-hover: #782040;
-  --abx-paper: #fcfaf7;
-  --abx-ink: #28252a;
-  --abx-muted: #716c74;
-  --abx-line: #e9e2e3;
-  --font-sans: var(--abx-font-sans);
-  --font-mono: var(--abx-font-mono);
-  --ab-font-sans: var(--abx-font-sans);
-  --ab-font-serif: var(--abx-font-sans);
-  --ab-font-label: var(--abx-font-sans);
-  --ab-font-mono: var(--abx-font-mono);
-  --ab-accent: var(--abx-accent);
-  --ab-accent-hover: var(--abx-accent-hover);
-  --ab-on-accent: #fff;
-}
-body { font-family: var(--abx-font-sans); }
-code, pre, kbd, samp { font-family: var(--abx-font-mono); }
-/* Reuse existing CTA classes; platform selection continues to control colors. */
-.button, .ab-btn {
-  display: inline-flex; align-items: center; justify-content: center; gap: 10px;
-  min-height: 44px; height: auto; max-width: 100%; padding: 10px 20px;
-  border-radius: 24px; font-family: var(--abx-font-sans);
-  font-size: 14px; font-weight: 650; line-height: 1.4;
-  text-align: center; white-space: normal; text-decoration: none;
-}
-.button.primary, .ab-btn--primary { background: var(--abx-accent); border-color: var(--abx-accent); color: #fff; }
-.button.primary:hover, .ab-btn--primary:hover { background: var(--abx-accent-hover); border-color: var(--abx-accent-hover); color: #fff; }
-.button:focus-visible, .ab-btn:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
-.button svg, .button img, .ab-btn svg, .ab-btn img { flex-shrink: 0; }
-@media (prefers-reduced-motion: reduce) { .button, .ab-btn { transition: none; } .button:hover { transform: none; } }
-
-/* Shared ArchiveBox site chrome. Independent copy; edit alongside the other product sites, without a build dependency. */
-/* Native anchor navigation leaves room for the heading and its surrounding content. */
-html { scroll-behavior: smooth; scroll-padding-top: clamp(96px, 25vh, 240px); }
-@media (prefers-reduced-motion: reduce) { html { scroll-behavior: auto; } }
-.abx-header, .abx-footer { box-sizing: border-box; font-family: var(--abx-font-sans); font-size: 14px; line-height: 1.5; letter-spacing: normal; text-align: left; }
-.abx-header *, .abx-footer * { box-sizing: border-box; }
-.abx-header { position: sticky; top: 0; z-index: 100; display: flex; align-items: center; justify-content: space-between; gap: 24px; width: 100%; max-width: none; margin: 0; padding: 15px max(24px, calc((100% - 1280px) / 2)); border: 0; border-bottom: 1px solid #e9e2e3; background: #fcfaf7f5; color: #28252a; backdrop-filter: blur(16px); }
-.abx-header a, .abx-footer a { text-decoration: none; text-underline-offset: .24em; }
-.abx-header .abx-brand, .abx-footer .abx-brand { display: inline-flex; flex: 0 0 auto; align-items: center; gap: 11px; margin: 0; padding: 0; color: inherit; font-size: 19px; font-weight: 750; line-height: 1.2; letter-spacing: -.5px; white-space: nowrap; }
-.abx-brand .abx-logo { display: block; flex: 0 0 auto; width: 32px; height: 32px; margin: 0; }
-.abx-header .abx-subsite { transform: translateY(3px); margin-left: 2px; color: #716c74; font-size: 12px; font-weight: 450; letter-spacing: 0; }
-.abx-header .abx-nav { display: flex; align-items: center; justify-content: flex-end; flex-wrap: wrap; gap: 0; width: auto; max-width: none; margin: 0; padding: 0; font-size: 13px; font-weight: 600; }
-.abx-nav > a, .abx-nav > details { margin: 0 0 0 15px; padding: 0 0 0 15px; border: 0; border-left: 1px solid #ded4d8; border-radius: 0; background: none; }
-.abx-nav > :first-child { margin-left: 0; padding-left: 0; border-left: 0; }
-.abx-header .abx-nav a { display: inline-flex; align-items: center; min-height: 36px; color: #4d424b; white-space: nowrap; }
-.abx-header .abx-nav a:hover { color: #9b2854; }
-.abx-header .abx-nav .abx-cta { margin-left: 18px; padding: 9px 17px; border: 1px solid #9b2854; border-radius: 24px; background: #9b2854; color: #fff; }
-.abx-header .abx-nav .abx-cta:hover { background: #782040; color: #fff; }
-.abx-header .abx-apps { position: relative; }
-.abx-apps > summary { display: flex; align-items: center; gap: 7px; min-height: 36px; list-style: none; cursor: pointer; color: #4d424b; }
-.abx-apps > summary::-webkit-details-marker { display: none; }
-.abx-apps > summary::before { content: '›'; display: inline-block; color: #9b2854; font-size: 22px; line-height: 1; transition: transform .15s; }
-.abx-apps[open] > summary::before { transform: rotate(90deg); }
-.abx-header .abx-app-links { position: absolute; right: 0; top: calc(100% + 12px); width: 440px; max-width: calc(100vw - 24px); max-height: min(70vh, 580px); overflow-y: auto; padding: 8px; border: 1px solid #e9e2e3; border-radius: 16px; background: #fcfaf7; box-shadow: 0 18px 48px #2d152522, 0 3px 8px #2d152508; }
-.abx-header .abx-app-row { display: flex; align-items: center; gap: 8px; min-height: 44px; padding: 4px 6px; border-radius: 10px; transition: background .15s; }
-.abx-header .abx-app-row:hover, .abx-header .abx-app-row:focus-within { background: #f2e8ed; }
-.abx-header .abx-app-links .abx-app-name { flex: 1; min-width: 0; min-height: 34px; padding: 5px 4px; border-radius: 6px; font-size: 13px; line-height: 1.35; white-space: normal; }
-.abx-header .abx-platforms { display: flex; flex: 0 0 auto; align-items: center; gap: 5px; }
-.abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 auto; height: 30px; min-height: 30px; padding: 5px; border: 1px solid #e8dfe3; border-radius: 9px; background: #fffdfb; color: #4d424b; box-shadow: 0 2px 3px #2d152510, inset 0 1px 0 #fff; transition: transform .15s, box-shadow .15s, border-color .15s; }
-.abx-header .abx-app-links .abx-platform { width: 30px; }
-.abx-header .abx-platform svg { display: block; width: 18px; height: 18px; margin: 0; fill: currentColor; }
-.abx-header .abx-app-links .abx-platform-docker { color: #2386ce; }
-.abx-header .abx-app-links .abx-platform-windows, .abx-header .abx-app-links .abx-platform-app-store, .abx-header .abx-app-links .abx-platform-safari { color: #1879c4; }
-.abx-header .abx-app-links .abx-platform-android { color: #29965d; }
-.abx-header .abx-app-links .abx-platform-firefox { color: #e46b24; }
-.abx-header .abx-app-links .abx-platform-brave { color: #e8552b; }
-.abx-header .abx-app-links .abx-platform-edge { color: #168b95; }
-.abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-platform:focus-visible, .abx-header .abx-app-links .abx-tool:hover, .abx-header .abx-app-links .abx-tool:focus-visible { transform: translateY(-2px); border-color: #c99bb0; background: #fff; box-shadow: 0 5px 10px #2d15251c; }
-.abx-header .abx-app-links .abx-tool { padding: 5px 8px; font-size: 11px; white-space: nowrap; }
-.abx-header .abx-tools-row { margin-top: 5px; padding-top: 9px; border-top: 1px solid #e9e2e3; border-radius: 0 0 10px 10px; }
-.abx-header .abx-tools-row .abx-app-name { flex: 0 0 auto; }
-.abx-header .abx-tools-row .abx-platforms { flex: 0 1 auto; flex-wrap: wrap; justify-content: flex-end; margin-left: auto; }
-.abx-header .abx-app-links code { display: inline; vertical-align: baseline; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 1em; font-weight: inherit; line-height: inherit; margin: 0; padding: 0; border: 0; background: transparent; color: inherit; }
-.abx-header .abx-icon-stack { isolation: isolate; gap: 0; padding: 4px 5px; }
-.abx-header .abx-icon-stack .abx-integration { position: relative; z-index: 1; transform: translateY(var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); transition: transform .18s ease, box-shadow .18s ease, border-color .18s ease; }
-.abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -10px; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(2) { --stack-angle: 9deg; --stack-y: -2px; z-index: 2; color: #569a31; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(3) { --stack-angle: -6deg; --stack-y: 2px; z-index: 3; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(4) { --stack-angle: 7deg; --stack-y: -1px; z-index: 4; color: #f00; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(5) { --stack-angle: -10deg; --stack-y: 1px; z-index: 5; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(6) { --stack-angle: 6deg; --stack-y: -2px; z-index: 6; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(7) { --stack-angle: -5deg; z-index: 7; }
-.abx-header .abx-icon-stack .abx-integration:not(:hover, :focus-visible):has(~ .abx-integration:is(:hover, :focus-visible)) { transform: translate(-6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
-.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) ~ .abx-integration:not(:hover, :focus-visible) { transform: translate(6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
-.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { z-index: 20; transform: translateY(-7px) rotate(0deg) scale(1.12); border-color: #c99bb0; box-shadow: 0 8px 14px #2d15252b; }
-@media (hover: none) { .abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -6px; } }
-@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration { transition: none; } }
-.abx-footer { position: relative; z-index: 1; display: block; width: 100%; max-width: none; margin: 72px 0 0; padding: 60px max(24px, calc((100% - 1280px) / 2)) 24px; border: 0; border-top: 3px solid #7e284a; background: radial-gradient(ellipse at 12% 0, #54243b55, transparent 48%), linear-gradient(120deg, #201923, #14151b 75%); color: #ddd2dd; color-scheme: dark; }
-.abx-footer .abx-footer-intro { display: flex; align-items: flex-start; justify-content: space-between; gap: 32px; margin: 0 0 42px; }
-.abx-footer .abx-brand { color: #fff4ed; font-size: 24px; }
-.abx-footer .abx-brand .abx-logo { width: 40px; height: 40px; }
-.abx-footer .abx-tagline { max-width: 430px; margin: 12px 0 0; color: #b8acbb; font-size: 14px; line-height: 1.7; }
-.abx-footer .abx-footer-motto { margin: 2px 0 0; color: #d8b39f; font: italic 25px/1.45 Georgia, serif; text-align: right; }
-.abx-footer .abx-footer-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 42px; margin: 0; }
-.abx-footer .abx-footer-column { margin: 0; padding: 20px 0 0; border: 0; border-top: 1px solid #67505e66; background: none; }
-.abx-footer .abx-footer-column h2 { margin: 0 0 19px; padding: 0; border: 0; color: #e4acbb; font: 650 12px/1.4 var(--abx-font-sans); letter-spacing: .12em; text-transform: uppercase; }
-.abx-footer .abx-footer-column ul { display: block; list-style: none; margin: 0; padding: 0; }
-.abx-footer .abx-footer-column li { display: flex; align-items: center; gap: 10px; margin: 0; padding: 0; min-height: 36px; }
-.abx-footer .abx-footer-column li::before { content: none; }
-.abx-footer .abx-footer-column a { color: #ddd2dd; font-size: 13px; line-height: 1.55; }
-.abx-footer .abx-footer-column a:first-child { padding: 6px 0; }
-.abx-footer a:hover { color: #fff; text-decoration: underline; }
-.abx-footer .abx-source { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 28px; width: 28px; height: 28px; border: 1px solid transparent; border-radius: 50%; color: #a58d9f; }
-.abx-footer .abx-source:hover { color: #f3b5cb; border-color: #80516a; background: #ffffff08; }
-.abx-source svg { width: 15px; height: 15px; fill: currentColor; }
-.abx-footer .abx-footer-bottom { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px 24px; margin-top: 32px; padding-top: 21px; border-top: 1px solid #67505e66; color: #a99bab; font-size: 12px; }
-.abx-footer .abx-footer-bottom p { margin: 0; font-size: inherit; color: inherit; }
-.abx-footer .abx-footer-bottom a { color: #cebbc9; }
-.abx-footer .abx-footer-local { margin: 24px 0 0; padding: 16px 0 0; border: 0; border-top: 1px solid #67505e66; border-radius: 0; background: none; color: #baacbc; font-size: 12px; }
-.abx-footer-local > summary { cursor: pointer; color: #d6bfce; }
-.abx-footer-local > div { padding-top: 18px; }
-.abx-footer-local a { color: #e5aabd; }
-.abx-footer-local .brand, .abx-footer-local .footer-intro, .abx-footer-local .footer-grid, .abx-footer-local .footer-meta { color: inherit; }
-.abx-footer-local .footer-card { background: transparent; border-color: #67505e66; }
-.abx-footer-local h2, .abx-footer-local h3, .abx-footer-local strong { color: #e4d8e3; }
-.abx-footer-local .footer-link-copy span, .abx-footer-local .footer-card-subtitle { color: #baacbc; }
-.abx-header a:focus-visible, .abx-header summary:focus-visible, .abx-footer a:focus-visible, .abx-footer summary:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
-@media (max-width: 1100px) {
-  .abx-header { flex-wrap: wrap; gap: 8px 24px; padding: 12px 20px; }
-  .abx-header .abx-nav { justify-content: flex-start; }
-  .abx-header .abx-app-links { position: fixed; top: auto; left: 20px; right: auto; width: 440px; margin-top: 10px; }
-}
-@media (max-width: 700px) {
-  .abx-header { position: relative; padding: 14px 18px; }
-  .abx-header .abx-brand { flex-wrap: wrap; row-gap: 5px; }
-  .abx-header .abx-nav { width: 100%; gap: 4px 0; font-size: 12px; }
-  .abx-nav > a, .abx-nav > details { margin-left: 10px; padding-left: 10px; }
-  .abx-header .abx-nav .abx-cta { margin-left: 10px; padding: 7px 13px; }
-  .abx-header .abx-app-links { left: 12px; right: 12px; width: auto; max-height: 60vh; }
-  .abx-footer { padding: 40px 22px 22px; margin-top: 48px; }
-  .abx-footer .abx-footer-intro { display: block; margin-bottom: 30px; }
-  .abx-footer .abx-footer-motto { margin-top: 20px; text-align: left; font-size: 22px; }
-  .abx-footer .abx-footer-grid { grid-template-columns: 1fr; gap: 26px; }
-  .abx-footer .abx-footer-column li { min-height: 40px; }
-}
-@media (prefers-reduced-motion: reduce) { .abx-apps > summary::before, .abx-header .abx-app-row, .abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { transition: none; } .abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-tool:hover { transform: none; } }
-
-.abx-footer-local > div > a + a { margin-left: 12px; }
-
-@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { transform: none; } }

.github/pages/base/chrome.js

@@ -1,28 +0,0 @@
-/*
-Shared ArchiveBox website navigation styles and behavior.
-Keep this file in sync with .github/pages/base/chrome.js in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
-*/
-(() => {
-  const menu = document.querySelector('.abx-apps');
-  document.addEventListener('keydown', event => {
-    if (event.key === 'Escape' && menu?.open) { menu.open = false; menu.querySelector('summary').focus(); }
-  });
-  document.addEventListener('click', event => { if (menu?.open && !menu.contains(event.target)) menu.open = false; });
-  function revealAnchor() {
-    let target;
-    try { target = document.getElementById(decodeURIComponent(window.location.hash.slice(1))); } catch { return; }
-    if (!target) return;
-    let details = target.closest('details');
-    let opened = false;
-    while (details) { if (!details.open) { details.open = true; opened = true; } details = details.parentElement.closest('details'); }
-    if (opened) window.requestAnimationFrame(() => target.scrollIntoView());
-  }
-  window.addEventListener('hashchange', revealAnchor);
-  revealAnchor();
-})();

.github/pages/base/footer-end.html

@@ -1,12 +0,0 @@
-<!--
-Shared ArchiveBox website footer.
-Keep this file in sync with .github/pages/base/footer-end.html in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
--->
-  <div class="abx-footer-bottom"><p>Built in the open. Supported by our community.</p><p><a href="https://github.com/ArchiveBox">GitHub organization</a> · <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Security &amp; privacy</a> · <a href="https://archivebox.io">ArchiveBox.io ↗</a></p></div>
-</footer>

.github/pages/base/footer-start.html

@@ -1,13 +0,0 @@
-<!--
-Shared ArchiveBox website footer.
-Keep this file in sync with .github/pages/base/footer-start.html in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
--->
-<footer class="abx-footer" aria-label="ArchiveBox ecosystem">
-  <div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
-  <div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated

.github/pages/base/header.html

@@ -1,24 +0,0 @@
-<!--
-Shared ArchiveBox website navbar and Apps menu.
-Keep this file in sync with .github/pages/base/header.html in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
--->
-<header class="abx-header">
-  <a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span><span class="abx-subsite">__SITE_NAME__</span></a>
-  <nav class="abx-nav" aria-label="Main navigation">
-    <details class="abx-apps"><summary>Apps</summary><div class="abx-app-links">
-      <div class="abx-app-row"><a class="abx-app-name" href="https://archivebox.io/#install-compose">ArchiveBox Server</a><div class="abx-platforms"><a class="abx-platform abx-platform-docker" href="https://archivebox.io/#install-compose" aria-label="ArchiveBox Server for Docker" title="ArchiveBox Server for Docker"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M349.9 236.3l-66.1 0 0-59.4 66.1 0 0 59.4zm0-204.3l-66.1 0 0 60.7 66.1 0 0-60.7zm78.2 144.8l-66.1 0 0 59.4 66.1 0 0-59.4zM271.8 104.7l-66.1 0 0 60.1 66.1 0 0-60.1zm78.1 0l-66.1 0 0 60.1 66.1 0 0-60.1zm276.8 100c-14.4-9.7-47.6-13.2-73.1-8.4-3.3-24-16.7-44.9-41.1-63.7l-14-9.3-9.3 14c-18.4 27.8-23.4 73.6-3.7 103.8-8.7 4.7-25.8 11.1-48.4 10.7l-434.7 0c-8.7 50.8 5.8 116.8 44 162.1 37.1 43.9 92.7 66.2 165.4 66.2 157.4 0 273.9-72.5 328.4-204.2 21.4 .4 67.6 .1 91.3-45.2 1.5-2.5 6.6-13.2 8.5-17.1l-13.3-8.9zM115.6 176.8l-66 0 0 59.4 66.1 0 0-59.4-.1 0zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm-78.1-72.1l-66.1 0 0 60.1 66.1 0 0-60.1z"/></svg></a><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/#archivebox-serverapp" aria-label="ArchiveBox Server for macOS" title="ArchiveBox Server for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://electron.archivebox.io/" aria-label="ArchiveBox Server for Windows" title="ArchiveBox Server for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://archivebox.io/#install-apt" aria-label="ArchiveBox Server for Linux" title="ArchiveBox Server for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-1Line truncated
-      <div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Desktop</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Desktop for macOS" title="ArchiveBox Desktop for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://windows.archivebox.io/" aria-label="ArchiveBox Desktop for Windows" title="ArchiveBox Desktop for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://linux.archivebox.io/" aria-label="ArchiveBox Desktop for Linux" title="ArchiveBox Desktop for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-10 14.1-10.5 4.7-4.4 13.5-14.2 27.9-14.2 7.1 0 15.6 2.3 25.9 8.9 6.3 4.1 11.3 4.4 22.6 9.3 8.4 3.5 13.7 9.7 10.5 18.2-2.6 7.1-11 14.4-22.7 18.1-11.1 3.6-19.8 16-38.2 14.9-3.9-.2-7-1-9.6-2.1-8-3.5-12.2-10.4-20-15-8.6-4.8-13.2-10.4-14.7-15.3-1.4-4.9 0-9 4.2-12.3zm3.3 334c-2.7 35.1-43.9 34.4-75.3 18-29.9-15.8-68.6-6.5-76.5-21.9-2.4-4.7-2.4-12.7 2.6-26.4l0-.2c2.4-7.6 .6-16-.6-23.9-1.2-7.8-1.8-15 .9-20 3.5-6.7 8.5-9.1 14.8-11.3 10.3-3.7 11.8-3.4 19.6-9.9 5.5-5.7 9.5-12.9 14.3-18 5.1-5.5 10-8.1 17.7-6.9 8.1 1.2 15.1 6.8 21.9 16l19.6 35.6c9.5 19.9 43.1 48.4 41 68.9zm-1.4-25.9c-4.1-6.6-9.6-13.6-14.4-19.6 7.1 0 14.2-2.2 16.7-8.9 2.3-6.2 0-14.9-7.4-24.9-13.5-18.2-38.3-32.5-38.3-32.5-13.5-8.4-21.1-18.7-24.6-29.9s-3-23.3-.3-35.2c5.2-22.9 18.6-45.2 27.2-59.2 2.3-1.7 .8 3.2-8.7 20.8-8.5 16.1-24.4 53.3-2.6 82.4 .6-20.7 5.5-41.8 13.8-61.5 12-27.4 37.3-74.9 39.3-112.7 1.1 .8 4.6 3.2 6.2 4.1 4.6 2.7 8.1 6.7 12.6 10.3 12.4 10 28.5 9.2 42.4 1.2 6.2-3.5 11.2-7.5 15.9-9 9.9-3.1 17.8-8.6 22.3-15 7.7 30.4 25.7 74.3 37.2 95.7 6.1 11.4 18.3 35.5 23.6 64.6 3.3-.1 7 .4 10.9 1.4 13.8-35.7-11.7-74.2-23.3-84.9-4.7-4.6-4.9-6.6-2.6-6.5 12.6 11.2 29.2 33.7 35.2 59 2.8 11.6 3.3 23.7 .4 35.7 16.4 6.8 35.9 17.9 30.7 34.8-2.2-.1-3.2 0-4Line truncated
-      <div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Mobile</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Mobile for iOS" title="ArchiveBox Mobile for iOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-android" href="https://android.archivebox.io/" aria-label="ArchiveBox Mobile for Android" title="ArchiveBox Mobile for Android"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!-- Font Awesome Free 5.0.13 by Fonticons, Inc. https://fontawesome.com License: https://creativecommons.org/licenses/by/4.0/ --><path d="M89.6 204.5v115.8c0 15.4-12.1 27.7-27.5 27.7-15.3 0-30.1-12.4-30.1-27.7V204.5c0-15.1 14.8-27.5 30.1-27.5 15.1 0 27.5 12.4 27.5 27.5zm10.8 157c0 16.4 13.2 29.6 29.6 29.6h19.9l.3 61.1c0 36.9 55.2 36.6 55.2 0v-61.1h37.2v61.1c0 36.7 55.5 36.8 55.5 0v-61.1h20.2c16.2 0 29.4-13.2 29.4-29.6V182.1H100.4v179.4zm248-189.1H99.3c0-42.8 25.6-80 63.6-99.4l-19.1-35.3c-2.8-4.9 4.3-8 6.7-3.8l19.4 35.6c34.9-15.5 75-14.7 108.3 0L297.5 34c2.5-4.3 9.5-1.1 6.7 3.8L285.1 73c37.7 19.4 63.3 56.6 63.3 99.4zm-170.7-55.5c0-5.7-4.6-10.5-10.5-10.5-5.7 0-10.2 4.8-10.2 10.5s4.6 10.5 10.2 10.5c5.9 0 10.5-4.8 10.5-10.5zm113.4 0c0-5.7-4.6-10.5-10.2-10.5-5.9 0-10.5 4.8-10.5 10.5s4.6 10.5 10.5 10.5c5.6 0 10.2-4.8 10.2-10.5zm94.8 60.1c-15.1 0-27.5 12.1-27.5 27.5v115.8c0 15.4 12.4 27.7 27.5 27.7 15.4 0 30.1-12.4 30.1-27.7V204.5c0-15.4-14.8-27.5-30.1-27.5z"/></svg></a></div></div>
-      <div class="abx-app-row"><a class="abx-app-name" href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><div class="abx-platforms"><a class="abx-platform abx-platform-chrome" href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj" aria-label="ArchiveBox Browser Extension for Chrome" title="ArchiveBox Browser Extension for Chrome"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc. --><path d="M0 256C0 209.4 12.47 165.6 34.27 127.1L144.1 318.3C166 357.5 207.9 384 256 384C270.3 384 283.1 381.7 296.8 377.4L220.5 509.6C95.9 492.3 0 385.3 0 256zM365.1 321.6C377.4 302.4 384 279.1 384 256C384 217.8 367.2 183.5 340.7 160H493.4C505.4 189.6 512 222.1 512 256C512 397.4 397.4 511.1 256 512L365.1 321.6zM477.8 128H256C193.1 128 142.3 172.1 130.5 230.7L54.19 98.47C101 38.53 174 0 256 0C350.8 0 433.5 51.48 477.8 128V128zM168 256C168 207.4 207.4 168 256 168C304.6 168 344 207.4 344 256C344 304.6 304.6 344 256 344C207.4 344 168 304.6 168 256z"/></svg></a><a class="abx-platform abx-platform-safari" href="https://extension.archivebox.io/" aria-label="ArchiveBox Browser Extension for Safari" title="ArchiveBox Browser Extension for Safari"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M274.7 274.7L237.3 237.3 166 346 274.7 274.7zM256 8a248 248 0 1 0 0 496 248 248 0 1 0 0-496zM411.8 182.8l14.8-6.1c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.8 1 2.6 1.7 1.3 1.6 1.7 2.6 .6 2 .6 3.1-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.3-2.6 1.7L418 197.6c-2 .8-4.2 .8-6.1 0s-3.5-2.4-4.3-4.3-.8-4.2 0-6.1 2.4-3.5 4.3-4.3zM314.4 94l6.1-14.8c.8-2 2.4-3.5 4.3-4.3s4.2-.8 6.1 0 3.5 2.4 4.3 4.3 .8 4.2 0 6.1l-6.1 14.8c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0c-1-.4-1.9-1-2.6-1.7s-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1zM256 60c2.1 0 4.2 .8 5.7 2.3S264 65.9 264 68l0 16c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7S253.9 60 256 60zM181 74.9c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.9 1 2.6 1.7 1.3 1.6 1.7 2.6L197.6 94c.4 1 .7 2 .7 3.1s-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.4-2.6 1.8-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.8-1.3-1.7-1.7-2.7l-6.1-14.8c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3zm-63.6 42.5c1.5-1.5 3.5-2.3 5.7-2.3s4.2 .8 5.7 2.3L140 128.7c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3l-11.3-11.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6l0 0zM60 256c0-2.1 .8-4.2 2.3-5.7S65.9 248 68 248l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3l-16 0c-2.1 0-4.2-.8-5.7-2.3S60 258.1 60 256zm40.2 73.2l-14.8 6.1c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1 1-1.9 1.7-2.6 1.6-1.3 2.6-1.7L94 314.4c2-.8 4.2-.8 6.1 0s3.5 2.4 4.3 4.3 .8 4.2 0 6.1-2.4 3.5-4.3 4.3zm4.3-136c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1 2.4-3.5 4.3-4.3 4.2-.8 6.1 0l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1l0 0zM197.6 418l-6.1 14.8c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1l6.1-14.8c.4-1 1-1.9 1.7-2.7s1.6-1.4 2.6-1.8 2-.6 3.1-.6 2.1 .2 3.1 .6 1.9 1 2.6 1.8 1.3 1.7 1.7 2.6 .6 2 .6 3.1-.2 2.1-.7 3.1zM264 444c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7s3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3 2.3 3.5 2.3 5.7l0 16zm67-6.9c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6L314.4 418c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3 4.2-.8 6.1 0 3.5 2.4 4.3 4.3l6.1 14.8c.8 2 .8 4.2 0 6.1s-2.4 3.5-4.3 4.3zm63.6-42.5c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3L372 383.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6 3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3l11.3 11.3c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6l0 0zM286.2 286.2L110.3 401.7 225.8 225.8 401.7 110.3 286.2 286.2zM437.1 331c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1c.4-1 1-1.9 1.7-2.6s1.6-1.3 2.6-1.7 2-.6 3.1-.6 2.1 .2 3.1 .6l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1zm6.9-67l-16 0c-2.1 0-4.2-.8-5.7-2.3s-2.3-3.5-2.3-5.7 .8-4.2 2.3-5.7 3.5-2.3 5.7-2.3l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3z"/></svg></a><a class="abx-platform abx-platform-firefox" href="https://addons.mozilla.org/firefox/addon/archivebox-exporter/" aria-label="ArchiveBox Browser Extension for Firefox" title="ArchiveBox Browser Extension for Firefox"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512Line truncated
-      <div class="abx-app-row abx-plugin-row"><a class="abx-app-name" href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><div class="abx-platforms abx-icon-stack"><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Google Drive storage" title="Google Drive storage"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="m38.7 419.3 22.6 39c4.7 8.2 11.4 14.7 19.4 19.4l80.6-139.6H0c0 9.1 2.3 18.2 7 26.4z" style="fill:#0066da"/><path d="M256 173.9 175.4 34.3c-7.9 4.7-14.7 11.1-19.4 19.4L7 311.7c-4.6 8-7 17.1-7 26.4h161.3z" style="fill:#00ac47"/><path d="M431.4 477.7c7.9-4.7 14.7-11.1 19.4-19.4l9.4-16.1 44.9-77.7c4.7-8.2 7-17.3 7-26.4H350.7l34.3 67.4z" style="fill:#ea4335"/><path d="m256 173.9 80.6-139.6c-7.9-4.7-17-7-26.4-7H201.8c-9.4 0-18.5 2.6-26.4 7z" style="fill:#00832d"/><path d="M350.7 338.1H161.3L80.6 477.7c7.9 4.7 17 7 26.4 7h298c9.4 0 18.5-2.6 26.4-7z" style="fill:#2684fc"/><path d="M430.5 182.7 356 53.7c-4.7-8.2-11.4-14.7-19.4-19.4L256 173.9l94.7 164.2h161c0-9.1-2.3-18.2-7-26.4z" style="fill:#ffba00"/></svg></a><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Amazon S3 storage" title="Amazon S3 storage"><svg aria-hidden="true" focusable="false" fill="#569A31" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M20.913 13.147l.12-.895c.947.576 1.258.922 1.354 1.071-.16.031-.562.046-1.474-.176zm-2.174 7.988a.547.547 0 0 0-.005.073c0 .084-.207.405-1.124.768a10.28 10.28 0 0 1-1.438.432c-1.405.325-3.128.504-4.853.504-4.612 0-7.412-1.184-7.412-1.704a.547.547 0 0 0-.005-.073L1.81 5.602c.135.078.28.154.432.227.042.02.086.038.128.057.134.062.272.122.417.18l.179.069c.154.058.314.114.478.168.043.013.084.029.13.043.207.065.423.127.646.187l.176.044c.175.044.353.087.534.127a23.414 23.414 0 0 0 .843.17l.121.023c.252.045.508.085.768.122.071.011.144.02.216.03.2.027.4.053.604.077l.24.027c.245.026.49.05.74.07l.081.009c.275.022.552.04.83.056l.233.012c.21.01.422.018.633.025a33.088 33.088 0 0 0 2.795-.026l.232-.011c.278-.016.555-.034.83-.056l.08-.008c.25-.02.497-.045.742-.072l.238-.026c.205-.024.408-.05.609-.077.07-.01.141-.019.211-.03.261-.037.519-.078.772-.122l.111-.02c.215-.04.427-.082.634-.125l.212-.047c.186-.041.368-.085.546-.13l.166-.042c.225-.06.444-.122.654-.189.04-.012.077-.026.115-.038a10.6 10.6 0 0 0 .493-.173c.058-.021.114-.044.17-.066.15-.06.293-.12.43-.185.038-.017.079-.034.116-.052.153-.073.3-.15.436-.228l-.976 7.245c-2.488-.78-5.805-2.292-7.311-3a1.09 1.09 0 0 0-1.088-1.085c-.6 0-1.088.489-1.088 1.088 0 .6.488 1.089 1.088 1.089.196 0 .378-.056.537-.148 1.72.812 5.144 2.367 7.715 3.15zm-7.42-20.047c5.677 0 9.676 1.759 9.75 2.736l-.014.113c-.01.033-.031.067-.048.101-.015.028-.026.057-.047.087-.024.033-.058.068-.09.102-.028.03-.051.06-.084.09-.038.035-.087.07-.133.105-.04.03-.074.06-.119.091-.053.036-.116.071-.177.107-.05.03-.095.06-.15.09-.068.036-.147.073-.222.11-.059.028-.114.057-.177.085-.084.038-.177.074-.268.111-.068.027-.13.054-.203.082-.097.036-.205.072-.31.107-.075.026-.148.053-.228.079-.111.035-.233.069-.35.103-.085.024-.165.05-.253.073-.124.034-.258.065-.389.098-.093.022-.181.046-.278.068-.139.032-.287.061-.433.091-.098.02-.191.041-.293.06-.155.03-.32.057-.482.084-.1.018-.198.036-.302.052-.166.026-.342.048-.515.072-.11.014-.213.03-.325.044-.181.023-.372.041-.56.06-.11.012-.218.025-.332.036-.188.016-.386.029-.58.043-.122.009-.24.02-.364.028-.207.012-.422.02-.635.028-.12.005-.234.012-.354.016a35.605 35.605 0 0 1-2.069 0c-.12-.004-.234-.011-.352-.016-.214-.008-.43-.016-.637-.028-.122-.008-.238-.02-.36-.027-.195-.015-.394-.028-.584-.044-.11-.01-.215-.024-.324-.035-.19-.02-.384-.038-.568-.06l-.315-.044c-.176-.024-.355-.046-.525-.073-.1-.015-.192-.033-.29-.05-.167-.028-.335-.055-.494-.086-.096-.018-.183-.038-.276-.056-.151-.032-.305-.062-.45-.095-.09-.02-.173-.043-.26-.064-.138-.034-.277-.067-.407-.102-.082-.022-.157-.046-.235-.069a11.75 11.75 0 0 1-.368-.108c-.075-.024-.141-.049-.213-.073-.11-.037-.223-.075-.325-.113-.067-.025-.125-.051-.188-.077-.096-.038-.195-.076-.282-.115-.06-.027-.11-.054-.166-.08-.08-.039-.162-.077-.233-.116-.052-.028-.094-.055-.142-.084-.063-.038-.13-.075-.185-.113-.043-.029-.075-.058-.113-.086-.048-.037-.098-.073-.139-.11-.032-.029-.054-.057-.08-.087-.033-.035-.069-.07-.093-.104-.02-.03-.031-.058-.046-.086-.018-.035-.039-.068-.049-.102l-.015-.113c.076-.977 4.074-2.736 9.748-2.736zm12.182 12.124c-.118-.628-.84-1.291-2.31-2.128l.963-7.16a.531.531 0 0 0 .005-.073C22.16 1.581 16.447 0 11.32 0 6.194 0 .482 1.581.482 3.851a.58.58 0 0 0 .005.072L2.819 21.25c.071 2.002 5.236 2.75 8.5 2.75 1.805 0 3.615-.188 5.098-.531.598-.138 1.133-.3 1.592-.48 1.18-.467 1.789-1.053 1.813-1.739l.945-7.018c.557.131 1.016.197 1.389.197.54 0 .902-.137 1.134-.413aLine truncated
-      <div class="abx-app-row abx-tools-row"><a class="abx-app-name" href="https://abx-dl.archivebox.io/">Tools</a><div class="abx-platforms"><a class="abx-tool" href="https://abx-dl.archivebox.io/"><span>One-shot <code>abx-dl</code> CLI</span></a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/tree/dev/archivebox/mcp">MCP</a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-api-usage">API</a><a class="abx-tool" href="https://web.archivebox.io">Demo</a></div></div>
-    </div></details>
-    __SITE_NAV__
-  </nav>
-</header>

.github/pages/base/marquee.css

@@ -1,11 +0,0 @@
-/* Identical standalone gallery strip across ArchiveBox product sites. */
-/* The root query container measures the usable width, excluding scrollbars. */
-html { container-type: inline-size; }
-.abx-marquee { min-width: 0; width: 100cqw; max-width: none; margin: 24px calc(50% - 50cqw) 40px; padding: 0; overflow: hidden; }
-.abx-marquee .abx-marquee-viewport { width: 100%; overflow-x: auto; scroll-behavior: auto; overscroll-behavior-x: contain; scrollbar-width: thin; padding: 16px 24px 28px; box-sizing: border-box; }
-.abx-marquee .abx-marquee-track { display: flex; align-items: flex-start; gap: 24px; width: max-content; margin-inline: auto; }
-.abx-marquee .abx-marquee-card { display: block; flex: none; width: min-content; overflow: hidden; border-radius: 10px; background: #fff; box-shadow: 0 0 0 1px rgb(0 0 0 / 12%), 0 2px 4px rgb(0 0 0 / 10%), 0 8px 16px rgb(0 0 0 / 16%); text-decoration: none; color: #533344; }
-.abx-marquee .abx-marquee-card img { display: block; height: clamp(240px, 30vw, 400px); width: auto; max-width: none; max-height: none; object-fit: contain; box-shadow: none; border: 0; border-radius: 0; background: transparent; padding: 0; margin: 0; }
-.abx-marquee :focus-visible { outline: 2px solid #9b2854; outline-offset: 3px; }
-.abx-marquee .abx-marquee-viewport:focus-visible { outline-offset: -3px; }
-@media (max-width: 600px) { .abx-marquee .abx-marquee-track { gap: 16px; } .abx-marquee .abx-marquee-card img { height: 240px; } }

.github/pages/base/marquee.js

@@ -1,44 +0,0 @@
-/* Scroll real gallery links once; keep them usable without JavaScript. */
-(() => {
-  document.querySelectorAll('.abx-marquee').forEach(gallery => {
-    const viewport = gallery.querySelector('.abx-marquee-viewport');
-    const reducedMotion = window.matchMedia('(prefers-reduced-motion: reduce)');
-    let paused = reducedMotion.matches;
-    let hovering = false;
-    let focused = false;
-    let visible = false;
-    let frame;
-    let previous;
-    let offset = viewport.scrollLeft;
-    function tick(time) {
-      const distance = viewport.scrollWidth - viewport.clientWidth;
-      if (distance <= 0) return;
-      if (previous !== undefined) {
-        offset = Math.min(offset + Math.min(time - previous, 50) * 0.035, distance);
-        viewport.scrollTo({ left: offset, behavior: 'instant' });
-      }
-      if (offset >= distance - 1) { paused = true; return; }
-      previous = time;
-      frame = window.requestAnimationFrame(tick);
-    }
-    function sync() {
-      window.cancelAnimationFrame(frame);
-      previous = undefined;
-      offset = viewport.scrollLeft;
-      if (!paused && !hovering && !focused && visible && !document.hidden) frame = window.requestAnimationFrame(tick);
-    }
-    viewport.addEventListener('pointerenter', event => { if (event.pointerType === 'mouse') { hovering = true; sync(); } });
-    viewport.addEventListener('pointerleave', () => { hovering = false; sync(); });
-    viewport.addEventListener('focusin', () => { focused = true; sync(); });
-    viewport.addEventListener('focusout', event => { focused = viewport.contains(event.relatedTarget); sync(); });
-    function pauseForInteraction() { paused = true; sync(); }
-    viewport.addEventListener('pointerdown', pauseForInteraction);
-    viewport.addEventListener('wheel', pauseForInteraction, { passive: true });
-    viewport.addEventListener('keydown', pauseForInteraction);
-    reducedMotion.addEventListener('change', () => { paused = reducedMotion.matches; sync(); });
-    document.addEventListener('visibilitychange', sync);
-    new window.IntersectionObserver(([entry]) => { visible = entry.isIntersecting; sync(); }).observe(gallery);
-    new window.ResizeObserver(sync).observe(viewport);
-    viewport.querySelectorAll('img').forEach(image => image.addEventListener('load', sync));
-  });
-})();

.github/pages/marquee.py

@@ -1,96 +0,0 @@
-"""Render homepage screenshot strips from the site's existing gallery HTML."""
-
-import re
-from html import escape
-from html.parser import HTMLParser
-from urllib.parse import urljoin
-
-
-class Gallery(HTMLParser):
-    """Only gallery figures are captures; navigation and decorative images are not."""
-
-    def __init__(self, base):
-        super().__init__()
-        self.base = base
-        self.stack = []
-        self.captures = []
-
-    def handle_starttag(self, tag, attrs):
-        attrs = dict(attrs)
-        if tag == "img" and any(name == "figure" for name, _ in self.stack):
-            context = [item for _, item in self.stack]
-            profile = next(
-                (
-                    item.get("data-profile") or item.get("data-viewport")
-                    for item in reversed(context)
-                    if item.get("data-profile") or item.get("data-viewport")
-                ),
-                None,
-            )
-            if profile not in (None, "desktop"):
-                return
-            source = attrs.get("src", "")
-            if not source or not attrs.get("alt"):
-                return
-            anchor = next((item["id"] for name, item in reversed(self.stack) if name in ("article", "figure") and item.get("id")), "")
-            server = any(item.get("data-platform") == "server" for item in context)
-            self.captures.append(
-                {
-                    "src": urljoin(self.base, source),
-                    "href": self.base + "#" + anchor if anchor else urljoin(self.base, source),
-                    "title": attrs["alt"],
-                    "width": attrs.get("width", ""),
-                    "height": attrs.get("height", ""),
-                    "product": "server" if server else "client",
-                },
-            )
-        if tag not in {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}:
-            self.stack.append((tag, attrs))
-
-    def handle_endtag(self, tag):
-        for index in range(len(self.stack) - 1, -1, -1):
-            if self.stack[index][0] == tag:
-                del self.stack[index:]
-                break
-
-
-def render(output, baseurl="", featured=()):
-    homepage = output / "index.html"
-    text = homepage.read_text()
-    slots = re.findall(r"<!-- ARCHIVEBOX:(MARQUEE(?:-CLIENT|-SERVER)?) -->", text)
-    if not slots:
-        return False
-    base = "/" + baseurl.strip("/") + "/" if baseurl.strip("/") else "/"
-    gallery = Gallery(base + "screenshots/")
-    gallery.feed((output / "screenshots/index.html").read_text())
-    for slot in slots:
-        product = slot.removeprefix("MARQUEE-").lower() if "-" in slot else None
-        captures = [capture for capture in gallery.captures if product is None or capture["product"] == product]
-        priority = {title: index for index, title in enumerate(featured)}
-        captures.sort(key=lambda capture: priority.get(capture["title"], len(priority)))
-        cards = []
-        for capture in captures:
-            dimensions = "".join(f' {key}="{escape(capture[key])}"' for key in ("width", "height") if capture[key])
-            cards.append(
-                f'<a class="abx-marquee-card" href="{escape(capture["href"])}"><img src="{escape(capture["src"])}" alt="{escape(capture["title"])}"{dimensions} loading="lazy" decoding="async"></a>',
-            )
-        viewport_id = slot.lower()
-        content = (
-            f'<section class="abx-marquee" aria-label="App screenshots"><div class="abx-marquee-viewport" id="{viewport_id}" tabindex="0" aria-label="Screenshots. Scroll sideways to explore."><div class="abx-marquee-track">{"".join(cards)}</div></div></section>'
-            if cards
-            else ""
-        )
-        start, end = f"<!-- ARCHIVEBOX:{slot} -->", f"<!-- /ARCHIVEBOX:{slot} -->"
-        replacement = start + "\n" + content + "\n" + end
-        if end in text:
-            text = re.sub(
-                re.escape(start) + r".*?" + re.escape(end),
-                lambda _, replacement=replacement: replacement,
-                text,
-                count=1,
-                flags=re.DOTALL,
-            )
-        else:
-            text = text.replace(start, replacement, 1)
-    homepage.write_text(text)
-    return True

.github/pages/nav.html

@@ -1,4 +0,0 @@
-<!-- Repository-specific navigation and primary CTA. -->
-<a href="__SITE_BASE__#use-cases">Use cases</a><a href="__SITE_BASE__#configuration">Configuration</a><a href="__SITE_BASE__#layout-security">Layout &amp; security</a><a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
-    <a href="https://github.com/ArchiveBox/ArchiveBox">GitHub ↗</a>
-    <a class="abx-cta" href="__SITE_BASE__#install">Get started</a>

.github/pages/site.json

@@ -1,16 +0,0 @@
-{
-  "name": "Web Archive",
-  "url": "https://archivebox.io/",
-  "pages": [
-    "index.html",
-    "screenshots/index.html"
-  ],
-  "marquee_featured": [
-    "Snapshots grid — desktop",
-    "Snapshot View (header collapsed) — desktop",
-    "AI agent — desktop",
-    "Archive results — desktop",
-    "Add URLs — desktop",
-    "Snapshots table — desktop"
-  ]
-}

.github/pages/site.py

@@ -1,215 +0,0 @@
-"""Assemble and verify this repository's static Pages site (standard library only)."""
-
-import argparse
-import json
-import re
-import shutil
-import subprocess
-from datetime import UTC, datetime
-from html import escape, unescape
-from html.parser import HTMLParser
-from pathlib import Path
-from urllib.parse import unquote, urlsplit
-
-from marquee import render as render_marquee
-
-HERE = Path(__file__).resolve().parent
-ROOT = HERE.parents[1]
-
-
-def replace_block(text, name, content):
-    start, end = f"<!-- ARCHIVEBOX:{name} -->", f"<!-- /ARCHIVEBOX:{name} -->"
-    pattern = re.escape(start) + r".*?" + re.escape(end)
-    if re.search(pattern, text, re.DOTALL):
-        return re.sub(
-            pattern,
-            lambda _: f"{start}\n{content}\n{end}",
-            text,
-            count=1,
-            flags=re.DOTALL,
-        )
-    if text.count(start) != 1:
-        raise ValueError(f"Expected one {name} placeholder")
-    return text.replace(start, f"{start}\n{content}\n{end}", 1)
-
-
-def render_metadata(text):
-    """Keep product copy in native templates; render one consistent SEO block."""
-    page = Page()
-    page.feed(text)
-    title = unescape(re.search(r"<title>(.*?)</title>", text, re.DOTALL)[1])
-    meta = page.metadata
-    description = meta.get("description", meta.get("og:description", ""))
-    image = meta.get("og:image", meta.get("twitter:image", ""))
-    if not description or not image or not page.canonical:
-        raise ValueError("Each page needs a description, social image and canonical")
-    values = {
-        "description": description,
-        "robots": "index,follow,max-image-preview:large",
-        "theme-color": "#9b2854",
-        "og:type": "website",
-        "og:site_name": meta.get("og:site_name", "ArchiveBox"),
-        "og:locale": meta.get("og:locale", "en_US"),
-        "og:url": page.canonical,
-        "og:title": title,
-        "og:description": description,
-        "og:image": image,
-        "og:image:type": meta.get("og:image:type", "image/png"),
-        "og:image:width": meta.get("og:image:width", "1200"),
-        "og:image:height": meta.get("og:image:height", "630"),
-        "og:image:alt": meta.get("og:image:alt", title),
-        "twitter:card": "summary_large_image",
-        "twitter:site": "@ArchiveBoxApp",
-        "twitter:title": title,
-        "twitter:description": description,
-        "twitter:image": image,
-        "twitter:image:alt": meta.get("og:image:alt", title),
-    }
-
-    def strip_managed_tag(match):
-        tag = Page()
-        tag.feed(match[0])
-        if tag.canonicals or any(key in values for key in tag.metadata):
-            return ""
-        # System fonts are shared across sites, without a network font dependency.
-        if "fonts.googleapis.com" in match[0] or "fonts.gstatic.com" in match[0]:
-            return ""
-        return match[0]
-
-    text = re.sub(r"<(?:meta|link)\b[^>]*>", strip_managed_tag, text)
-    text = re.sub(r"<title>.*?</title>", "", text, flags=re.DOTALL)
-    tags = [
-        f"<title>{escape(title)}</title>",
-        f'<link rel="canonical" href="{escape(page.canonical, quote=True)}">',
-    ]
-    for key, value in values.items():
-        attr = "property" if key.startswith("og:") else "name"
-        tags.append(f'<meta {attr}="{key}" content="{escape(value, quote=True)}">')
-    if "<!-- ARCHIVEBOX:METADATA -->" not in text:
-        text = text.replace("</head>", "<!-- ARCHIVEBOX:METADATA -->\n</head>", 1)
-    return replace_block(text, "METADATA", "\n".join(tags))
-
-
-def render(output, source=None, baseurl=""):
-    config = json.loads((HERE / "site.json").read_text())
-    output = output.resolve()
-    if source:
-        source = source.resolve()
-        if output == source or output in source.parents or source in output.parents:
-            raise ValueError("Build output must be separate from the source directory")
-        if output == ROOT or output in ROOT.parents:
-            raise ValueError("Build output must not replace a repository")
-        if output.exists():
-            shutil.rmtree(output)
-        shutil.copytree(source, output)
-    revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
-    assets = output / "site-base"
-    assets.mkdir(parents=True, exist_ok=True)
-    has_marquee = render_marquee(output, baseurl, config.get("marquee_featured", []))
-    common_assets = ["chrome.css", "chrome.js"] + (["marquee.css", "marquee.js"] if has_marquee else [])
-    for name in common_assets:
-        shutil.copyfile(HERE / "base" / name, assets / name)
-    for name in config["pages"]:
-        page = output / name
-        text = render_metadata(page.read_text())
-        base = "../" * (len(Path(name).parts) - 1) or "./"
-        header = (HERE / "base/header.html").read_text().replace("__SITE_NAME__", escape(config["name"]))
-        header = header.replace("__SITE_NAV__", (HERE / "nav.html").read_text().strip())
-        text = replace_block(text, "HEADER", header)
-        for kind in ["FOOTER-START", "FOOTER-END"]:
-            text = replace_block(text, kind, (HERE / "base" / f"{kind.lower()}.html").read_text().strip())
-        text = text.replace("__SITE_BASE__", base).replace("__SITE_REVISION__", revision)
-        resources = f'<link rel="stylesheet" href="{base}site-base/chrome.css?v={revision}">\n<script src="{base}site-base/chrome.js?v={revision}" defer></script>'
-        if has_marquee and name == "index.html":
-            resources += f'\n<link rel="stylesheet" href="{base}site-base/marquee.css?v={revision}">\n<script src="{base}site-base/marquee.js?v={revision}" defer></script>'
-        if "<!-- ARCHIVEBOX:ASSETS -->" not in text:
-            text = text.replace("</head>", "<!-- ARCHIVEBOX:ASSETS -->\n</head>", 1)
-        text = replace_block(text, "ASSETS", resources)
-        page.write_text(text)
-    # Site build provenance is independent of all original screenshot manifests.
-    provenance_path = output / "build.json"
-    provenance = json.loads(provenance_path.read_text()) if provenance_path.exists() else {}
-    provenance.update(
-        revision=revision,
-        generatedAt=datetime.now(UTC).isoformat(),
-        site=config["url"],
-    )
-    provenance_path.write_text(json.dumps(provenance, indent=2) + "\n")
-    (output / ".nojekyll").touch()
-    verify(output, baseurl)
-    print(f"Assembled {len(config['pages'])} pages at {output} from {revision}")
-
-
-class Page(HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self.ids = set()
-        self.metadata = {}
-        self.canonical = ""
-        self.resources = []
-        self.headers = self.footers = self.canonicals = 0
-
-    def handle_starttag(self, tag, attrs):
-        attrs = dict(attrs)
-        if tag == "meta":
-            for key in ["name", "property"]:
-                if attrs.get(key):
-                    self.metadata[attrs[key]] = attrs.get("content", "")
-        if tag == "link" and attrs.get("rel") == "canonical":
-            self.canonical = attrs["href"]
-        if attrs.get("id"):
-            self.ids.add(attrs["id"])
-        classes = attrs.get("class", "").split()
-        self.headers += "abx-header" in classes
-        self.footers += "abx-footer" in classes
-        self.canonicals += tag == "link" and attrs.get("rel") == "canonical"
-        if tag in ["img", "script"] and attrs.get("src"):
-            self.resources.append(attrs["src"])
-        if tag == "link" and attrs.get("rel") in [
-            "stylesheet",
-            "icon",
-            "apple-touch-icon",
-        ]:
-            self.resources.append(attrs["href"])
-
-
-def verify(output, baseurl=""):
-    config = json.loads((HERE / "site.json").read_text())
-    output = output.resolve()
-    prefix = "/" + baseurl.strip("/") if baseurl.strip("/") else ""
-    for name in config["pages"]:
-        path = output / name
-        text = path.read_text()
-        page = Page()
-        page.feed(text)
-        if (page.headers, page.footers, page.canonicals) != (1, 1, 1):
-            raise ValueError(f"{name}: expected exactly one shared header, footer and canonical")
-        if "__SITE_" in text:
-            raise ValueError(f"{name}: unresolved site placeholder")
-        for resource in page.resources:
-            url = urlsplit(resource)
-            if url.scheme or url.netloc or not url.path:
-                continue
-            resource_path = unquote(url.path)
-            if prefix and resource_path.startswith(prefix + "/"):
-                resource_path = resource_path[len(prefix) :]
-            local = (output / resource_path.lstrip("/") if resource_path.startswith("/") else path.parent / resource_path).resolve()
-            if not local.is_relative_to(output) or not local.is_file():
-                raise ValueError(f"{name}: missing local resource {resource}")
-        for asset in ["chrome.css", "chrome.js"]:
-            if (output / "site-base" / asset).read_bytes() != (HERE / "base" / asset).read_bytes():
-                raise ValueError(f"{name}: stale common asset {asset}")
-    print(f"Verified shared layout and local resources for {len(config['pages'])} pages")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("command", choices=["render", "verify"])
-    parser.add_argument("output", type=Path)
-    parser.add_argument("--source", type=Path)
-    parser.add_argument("--baseurl", default="")
-    args = parser.parse_args()
-    if args.command == "render":
-        render(args.output, args.source, args.baseurl)
-    else:
-        verify(args.output, args.baseurl)

.github/pages/verify.py

@@ -1,256 +0,0 @@
-"""Check the built site in real Chromium at desktop and phone widths."""
-
-import argparse
-import json
-import os
-import re
-from functools import partial
-from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
-from html.parser import HTMLParser
-from pathlib import Path
-from threading import Thread
-from urllib.parse import unquote, urljoin, urlsplit
-
-from playwright.sync_api import expect, sync_playwright
-
-HERE = Path(__file__).resolve().parent
-
-
-class Handler(SimpleHTTPRequestHandler):
-    def handle(self):
-        # Browser navigation can cancel an in-flight lazy image response.
-        try:
-            super().handle()
-        except (BrokenPipeError, ConnectionResetError):
-            pass
-
-    def log_message(self, *args):
-        pass
-
-
-class ScreenshotReferences(HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self.references = []
-        self.ids = set()
-
-    def handle_starttag(self, tag, attributes):
-        attrs = dict(attributes)
-        if attrs.get("id"):
-            self.ids.add(attrs["id"])
-        for attribute in ("src", "href"):
-            if attrs.get(attribute):
-                self.references.append(attrs[attribute])
-
-
-def verify_screenshot_references(output, canonical):
-    """Check gallery embeds on every static page, including optional translations."""
-    pages = {}
-    for path in output.rglob("*.html"):
-        page = ScreenshotReferences()
-        page.feed(path.read_text())
-        pages[path.resolve()] = page
-    for path, page in pages.items():
-        page_url = urljoin(canonical, path.relative_to(output.resolve()).as_posix())
-        for reference in page.references:
-            url = urlsplit(urljoin(page_url, reference))
-            if url.netloc != urlsplit(canonical).netloc or "/screenshots/" not in url.path:
-                continue
-            target = output / unquote(url.path).lstrip("/")
-            if target.is_dir():
-                target /= "index.html"
-            target = target.resolve()
-            assert target.is_relative_to(output.resolve()) and target.is_file(), f"{path}: missing screenshot target {reference}"
-            assert not re.match(r"^\d+-", target.name), f"{path}: screenshot filename depends on capture order: {reference}"
-            if url.fragment and target in pages:
-                assert unquote(url.fragment) in pages[target].ids, f"{path}: missing screenshot anchor {reference}"
-
-
-def verify(output, evidence):
-    config = json.loads((HERE / "site.json").read_text())
-    verify_screenshot_references(output, config["url"])
-    evidence.mkdir(parents=True, exist_ok=True)
-    server = ThreadingHTTPServer(("127.0.0.1", 0), partial(Handler, directory=str(output.resolve())))
-    thread = Thread(target=server.serve_forever, daemon=True)
-    thread.start()
-    origin = f"http://127.0.0.1:{server.server_port}"
-    try:
-        with sync_playwright() as playwright:
-            browser = playwright.chromium.launch(executable_path=os.environ.get("CHROME_BINARY"), ignore_default_args=["--hide-scrollbars"])
-            for name in config["pages"]:
-                route = name.removesuffix("index.html")
-                page = browser.new_page(reduced_motion="reduce")
-                errors = []
-                missing = []
-                page.on("pageerror", lambda error, errors=errors: errors.append(str(error)))
-                page.on(
-                    "response",
-                    lambda response, missing=missing: (
-                        missing.append(response.url) if response.url.startswith(origin) and response.status >= 400 else None
-                    ),
-                )
-                for width in [1716, 2560, 390]:
-                    page.set_viewport_size({"width": width, "height": 900})
-                    page.goto(f"{origin}/{route}", wait_until="domcontentloaded")
-                    header = page.locator(".abx-header")
-                    expect(header).to_have_count(1)
-                    expect(header).to_be_visible()
-                    expect(header).to_be_in_viewport()
-                    brand = header.locator(".abx-brand")
-                    expect(brand).to_be_visible()
-                    expect(brand).to_contain_text("ArchiveBox")
-                    expect(brand.locator(".abx-logo")).to_be_visible()
-                    expect(brand.locator(".abx-logo")).to_be_in_viewport()
-                    expect(header).to_have_css("display", "flex")
-                    expect(page.locator(".abx-footer")).to_have_count(1)
-                    expect(page.locator(".abx-footer-column")).to_have_count(3)
-                    expect(header.locator(".abx-cta")).to_be_visible()
-                    menu = header.locator(".abx-apps")
-                    menu.locator("summary").focus()
-                    page.keyboard.press("Enter")
-                    expect(menu).to_have_attribute("open", "")
-                    expect(menu.locator("a").first).to_be_visible()
-                    page.keyboard.press("Escape")
-                    expect(menu).not_to_have_attribute("open", "")
-                    menu.locator("summary").click()
-                    box = header.bounding_box()
-                    assert box
-                    page.mouse.click(box["x"] + 2, box["y"] + 5)
-                    expect(menu).not_to_have_attribute("open", "")
-                    assert page.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth"), (
-                        f"{name}: overflow at {width}"
-                    )
-                    if not route and "screenshots/index.html" in config["pages"]:
-                        strips = page.locator(".abx-marquee")
-                        assert strips.count() > 0, "Homepage must use its screenshot gallery"
-                        for strip in strips.all():
-                            strip.scroll_into_view_if_needed()
-                            bounds = strip.bounding_box()
-                            usable_width = page.evaluate("document.documentElement.clientWidth")
-                            assert bounds and abs(bounds["x"]) < 1
-                            assert abs(bounds["width"] - usable_width) < 1
-                            assert not strip.inner_text().strip(), "Screenshot strips should have no visible captions or links"
-                            cards = strip.locator(".abx-marquee-card")
-                            assert cards.count() > 0, "Screenshot strip is empty"
-                            expect(strip.locator("button")).to_have_count(0)
-                            expect(strip.locator("a:not(.abx-marquee-card)")).to_have_count(0)
-                            viewport = strip.locator(".abx-marquee-viewport")
-                            overflowing = viewport.evaluate(
-                                "node => node.scrollWidth > node.clientWidth",
-                            )
-                            if overflowing:
-                                page.mouse.move(0, 0)
-                                page.emulate_media(reduced_motion="no-preference")
-                                initial = viewport.evaluate("node => node.scrollLeft")
-                                page.wait_for_function(
-                                    "([node, initial]) => node.scrollLeft > initial",
-                                    arg=[viewport.element_handle(), initial],
-                                )
-                                viewport.hover()
-                                stopped = viewport.evaluate("node => node.scrollLeft")
-                                viewport.evaluate(
-                                    "node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
-                                )
-                                assert viewport.evaluate("node => node.scrollLeft") == stopped
-                                page.emulate_media(reduced_motion="reduce")
-                                page.mouse.move(0, 0)
-                                stopped = viewport.evaluate("node => node.scrollLeft")
-                                viewport.evaluate(
-                                    "node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
-                                )
-                                assert viewport.evaluate("node => node.scrollLeft") == stopped
-                                viewport.focus()
-                                page.keyboard.press("ArrowRight")
-                                assert viewport.evaluate("node => node.scrollWidth > node.clientWidth")
-                            else:
-                                centered = viewport.evaluate(
-                                    """node => {
-                                        const view = node.getBoundingClientRect();
-                                        const track = node.querySelector('.abx-marquee-track').getBoundingClientRect();
-                                        return Math.abs((track.left - view.left) - (view.right - track.right)) < 2;
-                                    }""",
-                                )
-                                assert centered, "Short screenshot strips must be centered"
-                            for card in cards.all():
-                                href = card.get_attribute("href")
-                                assert href and "/screenshots/" in href
-                                response = page.request.get(urljoin(page.url, href))
-                                assert response.ok, f"Broken screenshot link: {href}"
-                                anchor = urlsplit(href).fragment
-                                if anchor:
-                                    assert f'id="{anchor}"' in response.text(), f"Missing screenshot anchor: {href}"
-                    # Decode every local content image using the real browser loader,
-                    # including images in offscreen carousel tracks and lazy galleries.
-                    images = page.locator("main img").evaluate_all("nodes => [...new Set(nodes.map(image => image.src))]")
-                    local_images = [url for url in images if url.startswith(origin + "/")]
-                    page.evaluate(
-                        """async urls => {
-                        for (const url of urls) {
-                            const image = new Image();
-                            image.src = url;
-                            await image.decode();
-                            if (!image.naturalWidth) throw new Error(`Broken image: ${url}`);
-                        }
-                    }""",
-                        local_images,
-                    )
-                    page.evaluate("scrollTo({top: 0, behavior: 'instant'})")
-                    page.evaluate(
-                        "() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
-                    )
-                    page.screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}{width}.png"))
-                    page.locator(".abx-footer").screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}footer-{width}.png"))
-                    for link in header.locator(".abx-nav > a").all():
-                        url = urlsplit(link.get_attribute("href") or "")
-                        if url.scheme or url.netloc:
-                            continue
-                        response = page.request.get(urljoin(page.url, url.geturl()))
-                        assert response.ok, f"Broken navigation: {url.geturl()}"
-                        if url.fragment:
-                            body = response.text()
-                            assert f'id="{url.fragment}"' in body or f"id='{url.fragment}'" in body, (
-                                f"Missing navigation anchor: {url.geturl()}"
-                            )
-                    if route == "" and page.locator("#resources").count():
-                        page.goto(f"{origin}/#resources")
-                        expect(page.locator("#resources")).to_be_visible()
-                assert not errors, errors
-                assert not missing, missing
-                page.close()
-                context = browser.new_context(java_script_enabled=False, viewport={"width": 390, "height": 900})
-                plain = context.new_page()
-                plain.goto(f"{origin}/{route}", wait_until="domcontentloaded")
-                expect(plain.locator(".abx-header")).to_be_visible()
-                expect(plain.locator(".abx-header .abx-brand")).to_be_in_viewport()
-                expect(plain.locator(".abx-header .abx-logo")).to_be_visible()
-                plain.locator(".abx-apps summary").click()
-                expect(plain.locator(".abx-app-links a").first).to_be_visible()
-                expect(plain.locator(".abx-footer-column a").first).to_be_visible()
-                if not route and "screenshots/index.html" in config["pages"]:
-                    assert plain.locator(".abx-marquee-card").count() > 0
-                    for strip in plain.locator(".abx-marquee").all():
-                        bounds = strip.bounding_box()
-                        usable_width = plain.evaluate("document.documentElement.clientWidth")
-                        assert bounds and abs(bounds["x"]) < 1
-                        assert abs(bounds["width"] - usable_width) < 1
-                    assert plain.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth")
-                    expect(plain.locator(".abx-marquee button")).to_have_count(0)
-                    expect(plain.locator(".abx-marquee-card").first).to_be_visible()
-                context.close()
-                print(
-                    f"PASS {name}: desktop/mobile, keyboard, no-JS, images and resources",
-                    flush=True,
-                )
-            browser.close()
-    finally:
-        server.shutdown()
-        server.server_close()
-        thread.join()
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("output", type=Path)
-    parser.add_argument("--evidence", type=Path, default=Path("test-results/site"))
-    args = parser.parse_args()
-    verify(args.output, args.evidence)

.github/scripts/discover_test_matrix.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python3
-"""Discover every test once, batching measured short files."""
-
-import json
-import os
-from pathlib import Path
-
-
-def main() -> None:
-    root = Path(__file__).resolve().parents[2]
-    archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
-
-    if not archivebox_tests:
-        raise SystemExit("No ArchiveBox tests discovered")
-
-    # These are observed per-file pytest seconds, including fixture/startup
-    # time. Shared job setup is paid once per batch. Unknown/long files, optional
-    # dependencies and hosted-only requirements keep independent jobs.
-    durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
-    hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
-    requested = os.environ.get("TEST_FILE", "")
-    batches: list[list[str]] = []
-    batch: list[str] = []
-    batch_seconds = 0
-    for path in archivebox_tests:
-        test_path = path.relative_to(root).as_posix()
-        duration = durations.get(test_path, 60)
-        if duration >= 60 or path in hosted or path.stem == "test_auth_ldap" or test_path == requested:
-            batches.append([test_path])
-            continue
-        if batch and (len(batch) == 8 or batch_seconds + duration > 180):
-            batches.append(batch)
-            batch = []
-            batch_seconds = 0
-        batch.append(test_path)
-        batch_seconds += duration
-    if batch:
-        batches.append(batch)
-
-    matrix: list[dict[str, object]] = []
-    for paths in batches:
-        matrix.append(
-            {
-                "name": f"main/{Path(paths[0]).stem}" if len(paths) == 1 else f"batch/{Path(paths[0]).stem}",
-                "paths": paths,
-                "paths_arg": " ".join(paths),
-                "extra": "ldap" if paths == ["archivebox/tests/test_auth_ldap.py"] else "",
-                "count": len(paths),
-                "ugnas": False,
-            },
-        )
-
-    # Ordinary Linux tests can use either pool. Only genuine hosted-only
-    # requirements need a file header; filenames never enter runner policy.
-    # Keep a bounded NAS share so hosted capacity continues working in parallel.
-    # This is queued work, not a worker count: several waves keep the capped
-    # runners occupied while hosted jobs run in parallel.
-    nas_job_limit = int(os.environ.get("UGNAS_CI_MAX_JOBS", "3"))
-    # Keep long jobs early in the hosted queue. Giving the longest jobs to the
-    # CPU-capped NAS instead created a tail after hosted work had finished; use
-    # its extra slots for a bounded share of shorter work without filename rules.
-    matrix.sort(key=lambda item: sum(durations.get(path, 60) for path in item["paths"]), reverse=True)
-    eligible = [item for item in reversed(matrix) if not any(root / path in hosted for path in item["paths"])]
-    for item in eligible[:nas_job_limit]:
-        item["ugnas"] = True
-
-    discovered_paths = [path for entry in matrix for path in entry["paths"]]
-    if len(discovered_paths) != len(set(discovered_paths)):
-        raise SystemExit("Tests were not discovered exactly once")
-    if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
-        raise SystemExit("Discovered test shard coverage does not match test files")
-
-    print(
-        f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs; "
-        f"{sum(item['ugnas'] for item in matrix)} selected for ugNAS, subject to the availability check",
-    )
-    print(json.dumps(matrix, separators=(",", ":")))
-
-
-if __name__ == "__main__":
-    main()

.github/test-durations.json

@@ -1,176 +0,0 @@
-{
-  "source_runs": [
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37190995942",
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37237793977"
-  ],
-  "measurement": "Unbatched files use successful per-file pytest step wall seconds from run 37237793977, including process and fixture startup, on their actual hosted or ugNAS runner. Already-batched files retain prior JUnit measurements plus 9 seconds startup. Queue and shared job setup are excluded; unknown files remain independent.",
-  "seconds": {
-    "archivebox/tests/test_api_discovery_performance.py": 28,
-    "archivebox/tests/test_api_v1_auth_browser_session.py": 17,
-    "archivebox/tests/test_api_v1_auth_check_api_token.py": 15,
-    "archivebox/tests/test_api_v1_auth_get_api_token.py": 16,
-    "archivebox/tests/test_api_v1_cli_add.py": 52,
-    "archivebox/tests/test_api_v1_cli_remove.py": 33,
-    "archivebox/tests/test_api_v1_cli_schedule.py": 29,
-    "archivebox/tests/test_api_v1_cli_search.py": 15,
-    "archivebox/tests/test_api_v1_cli_update.py": 60,
-    "archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 59,
-    "archivebox/tests/test_api_v1_core_any_id.py": 13,
-    "archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 24,
-    "archivebox/tests/test_api_v1_core_archiveresults.py": 19,
-    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 106,
-    "archivebox/tests/test_api_v1_core_snapshots.py": 17,
-    "archivebox/tests/test_api_v1_core_snapshots_rss.py": 16,
-    "archivebox/tests/test_api_v1_core_tag_tag_id.py": 15,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 16,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 16,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 16,
-    "archivebox/tests/test_api_v1_core_tags.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 16,
-    "archivebox/tests/test_api_v1_core_tags_autocomplete.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_create.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_search.py": 17,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 69,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 15,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 15,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 16,
-    "archivebox/tests/test_api_v1_crawls_crawls.py": 16,
-    "archivebox/tests/test_api_v1_machine_binaries.py": 16,
-    "archivebox/tests/test_api_v1_machine_binary_binary_id.py": 16,
-    "archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 16,
-    "archivebox/tests/test_api_v1_machine_machine_current.py": 15,
-    "archivebox/tests/test_api_v1_machine_machine_machine_id.py": 16,
-    "archivebox/tests/test_api_v1_machine_machines.py": 15,
-    "archivebox/tests/test_api_v1_personas_personas.py": 15,
-    "archivebox/tests/test_api_v1_personas_sync.py": 16,
-    "archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 27,
-    "archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 19,
-    "archivebox/tests/test_archive_result_service.py": 49,
-    "archivebox/tests/test_auth_ldap.py": 37,
-    "archivebox/tests/test_binary_service.py": 46,
-    "archivebox/tests/test_cli_add_1.py": 75,
-    "archivebox/tests/test_cli_add_2.py": 98,
-    "archivebox/tests/test_cli_add_3.py": 93,
-    "archivebox/tests/test_cli_add_4.py": 94,
-    "archivebox/tests/test_cli_archiveresult.py": 48,
-    "archivebox/tests/test_cli_background_screenshot.py": 49,
-    "archivebox/tests/test_cli_binary.py": 12,
-    "archivebox/tests/test_cli_config.py": 88,
-    "archivebox/tests/test_cli_crawl.py": 81,
-    "archivebox/tests/test_cli_extract.py": 97,
-    "archivebox/tests/test_cli_extract_input.py": 84,
-    "archivebox/tests/test_cli_help.py": 16,
-    "archivebox/tests/test_cli_init.py": 111,
-    "archivebox/tests/test_cli_install.py": 38,
-    "archivebox/tests/test_cli_list.py": 116,
-    "archivebox/tests/test_cli_machine.py": 19,
-    "archivebox/tests/test_cli_manage.py": 32,
-    "archivebox/tests/test_cli_mcp.py": 30,
-    "archivebox/tests/test_cli_oneshot.py": 15,
-    "archivebox/tests/test_cli_persona.py": 56,
-    "archivebox/tests/test_cli_piping.py": 118,
-    "archivebox/tests/test_cli_pluginmap.py": 12,
-    "archivebox/tests/test_cli_process.py": 12,
-    "archivebox/tests/test_cli_remove.py": 46,
-    "archivebox/tests/test_cli_run_1.py": 93,
-    "archivebox/tests/test_cli_run_2.py": 111,
-    "archivebox/tests/test_cli_run_3.py": 51,
-    "archivebox/tests/test_cli_run_4.py": 78,
-    "archivebox/tests/test_cli_schedule.py": 59,
-    "archivebox/tests/test_cli_search.py": 25,
-    "archivebox/tests/test_cli_server.py": 126,
-    "archivebox/tests/test_cli_shell.py": 17,
-    "archivebox/tests/test_cli_snapshot.py": 55,
-    "archivebox/tests/test_cli_status.py": 46,
-    "archivebox/tests/test_cli_tag.py": 12,
-    "archivebox/tests/test_cli_update.py": 62,
-    "archivebox/tests/test_cli_update_reindex_snapshots.py": 84,
-    "archivebox/tests/test_cli_version.py": 81,
-    "archivebox/tests/test_config_DELETE_AFTER.py": 145,
-    "archivebox/tests/test_config_MAX_limits.py": 25,
-    "archivebox/tests/test_config_ONLY_NEW.py": 15,
-    "archivebox/tests/test_config_SAVE_TITLE.py": 59,
-    "archivebox/tests/test_config_URL_filters.py": 15,
-    "archivebox/tests/test_core_config.py": 11,
-    "archivebox/tests/test_core_forms.py": 14,
-    "archivebox/tests/test_crawl_history.py": 60,
-    "archivebox/tests/test_crawl_pause.py": 11,
-    "archivebox/tests/test_crawl_runner.py": 22,
-    "archivebox/tests/test_crawl_runner_2.py": 27,
-    "archivebox/tests/test_crawl_service.py": 19,
-    "archivebox/tests/test_frozen_crawl_config.py": 26,
-    "archivebox/tests/test_hook_result_lifecycle.py": 48,
-    "archivebox/tests/test_hooks.py": 54,
-    "archivebox/tests/test_issue_1896.py": 115,
-    "archivebox/tests/test_machine_models.py": 26,
-    "archivebox/tests/test_machine_service.py": 77,
-    "archivebox/tests/test_migrations_04_to_09.py": 25,
-    "archivebox/tests/test_migrations_07_to_09.py": 86,
-    "archivebox/tests/test_migrations_08_to_09.py": 85,
-    "archivebox/tests/test_migrations_fresh.py": 55,
-    "archivebox/tests/test_misc_checks.py": 12,
-    "archivebox/tests/test_misc_monkey_patches.py": 12,
-    "archivebox/tests/test_opencode_agent_1.py": 73,
-    "archivebox/tests/test_opencode_agent_2.py": 141,
-    "archivebox/tests/test_opencode_agent_3.py": 61,
-    "archivebox/tests/test_opencode_browser_1.py": 139,
-    "archivebox/tests/test_opencode_browser_2.py": 164,
-    "archivebox/tests/test_opencode_browser_3.py": 125,
-    "archivebox/tests/test_orm_helpers.py": 14,
-    "archivebox/tests/test_permissions.py": 12,
-    "archivebox/tests/test_persona_casing.py": 45,
-    "archivebox/tests/test_persona_runtime.py": 30,
-    "archivebox/tests/test_postgres_backend.py": 79,
-    "archivebox/tests/test_process_runtime_paths.py": 20,
-    "archivebox/tests/test_process_service.py": 40,
-    "archivebox/tests/test_proof_previews.py": 24,
-    "archivebox/tests/test_recursive_crawl_1.py": 361,
-    "archivebox/tests/test_recursive_crawl_2.py": 55,
-    "archivebox/tests/test_recursive_crawl_3.py": 67,
-    "archivebox/tests/test_release_notes.py": 12,
-    "archivebox/tests/test_release_source_fingerprint.py": 13,
-    "archivebox/tests/test_release_workflow.py": 15,
-    "archivebox/tests/test_resource_admission.py": 13,
-    "archivebox/tests/test_runner_deletion.py": 91,
-    "archivebox/tests/test_runner_watch.py": 75,
-    "archivebox/tests/test_search.py": 142,
-    "archivebox/tests/test_serve_static.py": 19,
-    "archivebox/tests/test_server_automatic_url.py": 22,
-    "archivebox/tests/test_server_security_browser.py": 121,
-    "archivebox/tests/test_settings_signal_webhooks.py": 13,
-    "archivebox/tests/test_setup_script.py": 12,
-    "archivebox/tests/test_shutdown_util.py": 13,
-    "archivebox/tests/test_snapshot_filesystem_migration.py": 16,
-    "archivebox/tests/test_snapshot_output_stacks.py": 63,
-    "archivebox/tests/test_snapshot_ref_lookup.py": 15,
-    "archivebox/tests/test_snapshot_service.py": 48,
-    "archivebox/tests/test_snapshot_url_length.py": 16,
-    "archivebox/tests/test_tag_service.py": 15,
-    "archivebox/tests/test_takeover_util_1.py": 60,
-    "archivebox/tests/test_takeover_util_2.py": 62,
-    "archivebox/tests/test_takeover_util_3.py": 76,
-    "archivebox/tests/test_takeover_util_4.py": 84,
-    "archivebox/tests/test_test_harness.py": 12,
-    "archivebox/tests/test_ui_add_view.py": 26,
-    "archivebox/tests/test_ui_add_view_runtime.py": 120,
-    "archivebox/tests/test_ui_admin_apitoken.py": 16,
-    "archivebox/tests/test_ui_admin_archiveresult.py": 18,
-    "archivebox/tests/test_ui_admin_config_widget.py": 13,
-    "archivebox/tests/test_ui_admin_crawl.py": 25,
-    "archivebox/tests/test_ui_admin_links.py": 23,
-    "archivebox/tests/test_ui_admin_machine.py": 18,
-    "archivebox/tests/test_ui_admin_persona.py": 19,
-    "archivebox/tests/test_ui_admin_snapshot.py": 138,
-    "archivebox/tests/test_ui_admin_tag.py": 16,
-    "archivebox/tests/test_ui_admin_user.py": 16,
-    "archivebox/tests/test_ui_config_views.py": 20,
-    "archivebox/tests/test_ui_live_progress.py": 121,
-    "archivebox/tests/test_ui_public_snapshot.py": 72,
-    "archivebox/tests/test_ui_savepagenow.py": 31,
-    "archivebox/tests/test_urls.py": 105,
-    "archivebox/tests/test_util.py": 13,
-    "archivebox/tests/test_version_metadata.py": 18,
-    "archivebox/tests/test_zip_preview.py": 29
-  }
-}

.github/workflows/ci.yml

@@ -1,103 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-    branches: [dev, main]
-  push:
-    branches: [dev, main]
-  workflow_dispatch:
-
-permissions:
-  actions: read
-  contents: read
-  packages: write
-  security-events: write
-
-concurrency:
-  group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }}
-  cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
-
-jobs:
-  prepare:
-    permissions:
-      contents: write
-    uses: ./.github/workflows/prepare-release.yml
-    secrets: inherit
-
-  lint:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Linters
-    uses: ./.github/workflows/lint.yml
-    secrets: inherit
-
-  install-cli-platform:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Install and CLI platform compatibility
-    uses: ./.github/workflows/test.yml
-    secrets: inherit
-
-  tests:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Discovered test matrix
-    uses: ./.github/workflows/test-parallel.yml
-    secrets: inherit
-
-  codeql:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: CodeQL
-    uses: ./.github/workflows/codeql.yml
-    secrets: inherit
-
-  python-artifacts:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Tested Python artifacts
-    uses: ./.github/workflows/pip.yml
-    secrets: inherit
-
-  docker-digests:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Tested Docker images
-    uses: ./.github/workflows/docker.yml
-    with:
-      push_digests: ${{ github.event_name == 'push' && (github.ref_name == 'dev' || github.ref_name == 'main') }}
-    secrets: inherit
-
-  required:
-    name: All required CI lanes
-    if: ${{ always() && needs.prepare.outputs.run_jobs == 'true' }}
-    needs:
-      - prepare
-      - lint
-      - install-cli-platform
-      - tests
-      - codeql
-      - python-artifacts
-      - docker-digests
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Verify every required lane succeeded
-        env:
-          LINT_RESULT: ${{ needs.lint.result }}
-          INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
-          TESTS_RESULT: ${{ needs.tests.result }}
-          CODEQL_RESULT: ${{ needs.codeql.result }}
-          PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
-          DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
-        run: |
-          for result in \
-            "$LINT_RESULT" \
-            "$INSTALL_CLI_PLATFORM_RESULT" \
-            "$TESTS_RESULT" \
-            "$CODEQL_RESULT" \
-            "$PYTHON_ARTIFACTS_RESULT" \
-            "$DOCKER_DIGESTS_RESULT"
-          do
-            [[ "$result" == success ]]
-          done
-          echo 'All required CI lanes and tested release artifacts succeeded.'

.github/workflows/claude.yml

@@ -17,24 +17,33 @@ jobs:
       (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
       (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
       (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       pull-requests: write
       issues: write
       id-token: write
-      actions: read
+      actions: read # Required for Claude to read CI results on PRs
     steps:
       - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - name: Run Claude Code
         id: claude
-        uses: anthropics/claude-code-action@b76a0776ae74036e77cd11018083743453d7ad35 # v1
+        uses: anthropics/claude-code-action@v1
         with:
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+
+          # This is an optional setting that allows Claude to read CI results on PRs
           additional_permissions: |
             actions: read
+
+          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
+          # prompt: 'Update the pull request description to include a summary of changes.'
+
+          # Optional: Add claude_args to customize behavior and configuration
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
           claude_args: '--allowed-tools Bash(gh pr:*)'

.github/workflows/codeql.yml

@@ -1,21 +1,46 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
 name: "CodeQL"
 
 on:
-  workflow_call:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
   schedule:
     - cron: '33 17 * * 6'
 
 concurrency:
-  group: codeql-${{ github.repository }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
   analyze:
     name: Analyze (${{ matrix.language }})
-    runs-on: ubuntu-24.04
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
     permissions:
+      # required for all workflows
       security-events: write
+
+      # required to fetch internal or private CodeQL packs
       packages: read
+
+      # only required for workflows in private repositories
       actions: read
       contents: read
 
@@ -25,18 +50,47 @@ jobs:
         include:
         - language: python
           build-mode: none
-        - language: javascript-typescript
-          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
     steps:
     - name: Checkout repository
-      uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      uses: actions/checkout@v4
 
+    # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/deploy-publicsite.yml

@@ -1,96 +1,43 @@
-name: Publish website
+name: Deploy Publicsite to GitHub Pages
 
 on:
   push:
-    branches: [dev]
-  pull_request:
+    branches:
+      - dev
+    paths:
+      - publicsite/**
+      - .github/workflows/deploy-publicsite.yml
   workflow_dispatch:
-  workflow_run:
-    workflows: ['Capture publicsite screenshots']
-    types: [completed]
-    branches: [dev]
 
 permissions:
   contents: read
-  actions: read
+  pages: write
+  id-token: write
 
-env:
-  # Repository-specific output; all publication logic below is a local copy.
-  SITE_OUTPUT: _site
-  DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-
-# One production queue covers restore, build and deploy. PRs have separate queues.
 concurrency:
-  group: pages-${{ github.event_name == 'pull_request' && github.ref || 'production' }}
-  cancel-in-progress: false
+  group: github-pages-publicsite
+  cancel-in-progress: true
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    outputs:
-      revision: ${{ steps.source.outputs.revision }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event_name == 'pull_request' && github.ref || env.DEFAULT_BRANCH }}
-          persist-credentials: false
-      - name: Record website source revision
-        id: source
-        run: echo "revision=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
-
-      # BEGIN REPOSITORY-SPECIFIC BUILD
-      - uses: astral-sh/setup-uv@v7
-      - name: Restore latest complete screenshots
-        run: uv run --no-project python bin/restore_site_screenshots.py publicsite/screenshots
-        env:
-          GH_TOKEN: ${{ github.token }}
-      - name: Render current gallery layout without recapturing images
-        run: uv run --no-project python bin/generate_ui_screenshot_gallery.py refresh publicsite/screenshots/index.html
-      - name: Assemble website with the local base components
-        run: uv run --no-project python .github/pages/site.py render _site --source publicsite
-      # END REPOSITORY-SPECIFIC BUILD
-
-      - name: Verify website in Chromium
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/site-browser
-        run: |
-          # Resolve the same supported browser as captures; Playwright is the
-          # driver only and must not provision a separate browser/runtime.
-          ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
-          CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
-          uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
-          CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
-          export CHROME_BINARY
-          test -x "$CHROME_BINARY"
-          uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py "$SITE_OUTPUT" --evidence "$RUNNER_TEMP/site-evidence"
-      - name: Save website verification evidence
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: site-verification
-          path: ${{ runner.temp }}/site-evidence
-          if-no-files-found: warn
-          retention-days: 14
-
-      - name: Upload website
-        uses: actions/upload-pages-artifact@v3
-        with:
-          path: ${{ env.SITE_OUTPUT }}
-
   deploy:
-    if: github.event_name != 'pull_request'
-    needs: build
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      pages: write
-      id-token: write
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/configure-pages@v5
-      - uses: actions/deploy-pages@v4
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./publicsite
+
+      - name: Deploy to GitHub Pages
         id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/docker.yml

@@ -1,44 +1,28 @@
 name: Build Docker image
 
 on:
-  workflow_call:
-    inputs:
-      push_digests:
-        description: Push validated per-platform images by digest for the release workflow
-        required: false
-        default: false
-        type: boolean
-      full_tests:
-        description: Run storage, CLI, and size validation in addition to release smoke checks
-        required: false
-        default: true
-        type: boolean
   workflow_dispatch:
-    inputs:
-      source_ref:
-        description: Exact 40-character commit SHA to build
-        required: true
-        type: string
-      push_digests:
-        description: Push validated per-platform images by digest for the release workflow
-        required: false
-        default: true
-        type: boolean
-      full_tests:
-        description: Run storage, CLI, and size validation in addition to release smoke checks
-        required: false
-        default: true
-        type: boolean
+  workflow_call:
+  push:
+    branches:
+      - "**"
+    tags:
+      - 'v*'
+  # pull_request:
 
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
   GHCR_IMAGE: ghcr.io/archivebox/archivebox
+  ABX_DL_IMAGE: archivebox/abx-dl:latest
 
 permissions:
-  actions: read
   contents: read
   packages: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     name: build ${{ matrix.platform }}
@@ -49,94 +33,62 @@ jobs:
         include:
           - platform: linux/amd64
             runner: ubuntu-24.04
+            cache_scope: docker-amd64
             artifact_name: digest-linux-amd64
-            local_tag: archivebox-ci:amd64
           - platform: linux/arm64
             runner: ubuntu-24.04-arm
+            cache_scope: docker-arm64
             artifact_name: digest-linux-arm64
-            local_tag: archivebox-ci:arm64
     steps:
       - name: Checkout
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          ref: ${{ inputs.source_ref || github.sha }}
+        uses: actions/checkout@v4
+        # with:
+        #   submodules: true
+        #   fetch-depth: 1
 
-      - name: Verify exact source commit
-        if: github.event_name == 'workflow_dispatch'
-        shell: bash
-        env:
-          SOURCE_REF: ${{ inputs.source_ref || github.sha }}
+      - name: Wait for released ArchiveBox deps on PyPI
         run: |
-          set -Eeuo pipefail
-          [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
-          [[ "$(git rev-parse HEAD)" == "$SOURCE_REF" ]]
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Resolve Docker through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python - <<'PY'
+          python3 - <<'PY'
+          import json
+          import re
+          import sys
+          import time
           import tomllib
-          from pathlib import Path
+          import urllib.request
 
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          matches = [package for package in packages if package["name"] == "abxpkg"]
-          if len(matches) != 1:
-              raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}")
-          wheel = matches[0]["wheels"][0]
-          print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
+                      releases = json.load(resp)["releases"]
+                  if version in releases:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
           PY
-          )"
-          test -n "$ABXPKG_SPEC"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_SPEC=$ABXPKG_SPEC" >> "$GITHUB_ENV"
-          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-
-          docker_env="$(
-            uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries"
-          )"
-          JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$docker_env" >> "$GITHUB_ENV"
-
-          docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
-          UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv"
-          {
-            echo "DOCKER_BINARY=$docker_binary"
-            echo "JQ_BINARY=$JQ_BINARY"
-            echo "UV_BINARY=$UV_BINARY"
-          } >> "$GITHUB_ENV"
-          for binary in "$docker_binary" "$JQ_BINARY" "$UV_BINARY"; do
-            test -L "$binary"
-            test -x "$binary"
-          done
-          "$docker_binary" --version
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+        uses: docker/setup-buildx-action@v3
         with:
+          version: latest
           install: true
 
       - name: Builder instance name
@@ -145,50 +97,33 @@ jobs:
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
-      - name: Verify published abx-dl image
+      - name: Wait for published abx-dl image
         id: abx_dl_image
         shell: bash
         run: |
           set -Eeuo pipefail
-          ABX_DL_VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import tomllib
-          from pathlib import Path
+          deadline=$((SECONDS + 1800))
+          until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
+            if (( SECONDS >= deadline )); then
+              echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
+              exit 1
+            fi
+            echo "${ABX_DL_IMAGE} is not published yet; waiting..."
+            sleep 30
+          done
 
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          matches = [package["version"] for package in packages if package["name"] == "abx-dl"]
-          if len(matches) != 1:
-              raise SystemExit(f"Expected one locked abx-dl package, found {len(matches)}")
-          print(matches[0])
-          PY
-          )"
-          test -n "$ABX_DL_VERSION"
-          ABX_DL_REFERENCE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
-          INSPECT_OUTPUT="$("$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_REFERENCE")"
-          printf '%s\n' "$INSPECT_OUTPUT"
-          ABX_DL_DIGEST="$(INSPECT_OUTPUT="$INSPECT_OUTPUT" "$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import os
-
-          digest = next(
-              (fields[1] for line in os.environ["INSPECT_OUTPUT"].splitlines() if (fields := line.split()) and fields[0] == "Digest:"),
-              "",
-          )
-          print(digest)
-          PY
-          )"
-          [[ "$ABX_DL_DIGEST" == sha256:* ]]
-          ABX_DL_IMAGE="${ABX_DL_REFERENCE%%@*}@${ABX_DL_DIGEST}"
           echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
+          docker buildx imagetools inspect "${ABX_DL_IMAGE}"
 
       - name: Login to Docker Hub
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        if: inputs.push_digests
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        if: inputs.push_digests
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -199,19 +134,18 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
+          VERSION="$(python3 - <<'PY'
           import tomllib
-          from pathlib import Path
 
-          print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
           PY
           )"
-          test -n "$VERSION"
 
           {
             echo 'labels<<EOF'
             echo "org.opencontainers.image.version=${VERSION}"
-            echo "org.opencontainers.image.revision=${{ inputs.source_ref || github.sha }}"
+            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
             echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
             echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
             echo 'EOF'
@@ -222,393 +156,180 @@ jobs:
 
       - name: Build and push digest
         id: docker_build
-        if: inputs.push_digests
-        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
+        uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile
           builder: ${{ steps.buildx.outputs.name }}
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: |
+            ${{ env.DOCKERHUB_IMAGE }}
+            ${{ env.GHCR_IMAGE }}
           labels: ${{ steps.docker_meta.outputs.labels }}
           build-args: |
             ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
-          pull: true
+          cache-from: type=gha,scope=${{ matrix.cache_scope }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
           platforms: ${{ matrix.platform }}
-          # Keep the released image as the one cache authority. A second GHA
-          # cache can hold gzip variants of identical layers, and BuildKit may
-          # prefer those over :dev's zstd blobs. That counter-intuitively makes
-          # a cache hit larger than the released image. :dev advances only
-          # after both architectures pass the release checks.
-          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
-          # Inline metadata reuses the released compressed blobs directly;
-          # force-compression would recompress the whole ~740 MiB image. The
-          # compressed-size gate below remains the regression guard.
-          cache-to: |
-            ${{ inputs.push_digests && 'type=inline' || '' }}
-          outputs: |
-            ${{ inputs.full_tests && format('type=oci,dest={0}/archivebox-image.tar,compression=zstd,compression-level=3', runner.temp) || '' }}
-            ${{ inputs.push_digests && format('type=image,"name={0},{1}",push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=3', env.DOCKERHUB_IMAGE, env.GHCR_IMAGE) || '' }}
-
-      - name: Build pull request image
-        id: docker_build_pr
-        if: ${{ !inputs.push_digests }}
-        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
-        with:
-          context: ./
-          file: ./Dockerfile
-          builder: ${{ steps.buildx.outputs.name }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-          build-args: |
-            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
-          pull: true
-          platforms: ${{ matrix.platform }}
-          # Test the same layer lineage users receive. Reusing :dev avoids a
-          # second cache whose compression can drift from the released image.
-          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
-          outputs: |
-            type=docker,name=${{ matrix.local_tag }}
-            type=oci,dest=${{ runner.temp }}/archivebox-image.tar
+          outputs: type=image,push-by-digest=true,name-canonical=true,push=true
 
       - name: Image digest
-        if: inputs.push_digests
         run: echo ${{ steps.docker_build.outputs.digest }}
 
-      - name: Verify optional agent install with a read-only browser profile
-        shell: bash
-        env:
-          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: |
-          set -Eeuo pipefail
-          PROFILE_DIR="$(mktemp -d)"
-          # shellcheck disable=SC2016
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' \
-            --volume "$PROFILE_DIR:/home/archivebox/.config/chromium:ro" \
-            "$IMAGE" bash -c '
-              set -Eeuo pipefail
-              test -w "$XDG_CONFIG_HOME"
-              test ! -w "$XDG_CONFIG_HOME/chromium"
-              uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_archivebox_install; verify_archivebox_install()"
-            '
-
-      - name: Validate exact built image version and commit
-        if: inputs.full_tests
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}"
-          SHORT_SHA="${{ (inputs.source_ref || github.sha) }}"
-          SHORT_SHA="${SHORT_SHA::7}"
-
-          DATA_DIR="$(mktemp -d)"
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
-          ARCHIVEBOX_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version)"
-          printf '%s\n' "$ARCHIVEBOX_VERSION_OUTPUT"
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
-          IMAGE_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt)"
-          ARCHIVEBOX_VERSION_OUTPUT="$ARCHIVEBOX_VERSION_OUTPUT" IMAGE_VERSION_OUTPUT="$IMAGE_VERSION_OUTPUT" \
-            "$UV_BINARY" run --no-cache --no-project python - \
-            "ArchiveBox v${{ steps.docker_meta.outputs.version }}" \
-            "COMMIT_HASH=${SHORT_SHA}" \
-            "COMMIT_HASH=${GITHUB_SHA}" <<'PY'
-          import os
-          import sys
-
-          version_output = os.environ["ARCHIVEBOX_VERSION_OUTPUT"]
-          image_version_lines = os.environ["IMAGE_VERSION_OUTPUT"].splitlines()
-          if sys.argv[1] not in version_output:
-              raise SystemExit(f"Missing version marker: {sys.argv[1]}")
-          if sys.argv[2] not in version_output:
-              raise SystemExit(f"Missing short commit marker: {sys.argv[2]}")
-          if sys.argv[3] not in image_version_lines:
-              raise SystemExit(f"Missing exact commit marker: {sys.argv[3]}")
-          PY
-
-          "$DOCKER_BINARY" run --rm --user 911:911 --platform '${{ matrix.platform }}' --entrypoint /venv/bin/python3 "$IMAGE" -c \
-            'import json, pathlib, stat; cache_file = "/opt/archivebox/lib/env/derived.env"; raw = open(cache_file).read().strip(); records = json.loads(raw[len("ABXPKG_DERIVED_CACHE=\x27"):-1]); projections = [projection for record in records.values() for projection in record.get("request_exec_projections", {}).values()]; plans = [projection["validation"] for projection in projections]; fingerprints = [fingerprint for plan in plans for fingerprint in plan["fingerprint"]]; actual = lambda fingerprint: (lambda path, value: {"path": str(path.resolve(strict=False)), "size": value.st_size, "mtime_ns": value.st_mtime_ns, "mode": stat.S_IMODE(value.st_mode), "euid": value.st_uid})(pathlib.Path(fingerprint["path"]), pathlib.Path(fingerprint["path"]).stat()); mismatches = [(fingerprint, actual(fingerprint)) for fingerprint in fingerprints if fingerprint != actual(fingerprint)]; assert projections and {plan.get("euid") for plan in plans} == {911}; assert all(not record.get("script_exec_plans") for record in records.values()); assert "ABXPKG_TMP_CACHE_DIR" not in json.dumps(projections); assert not mismatches, mismatches[:3]'
-
-      - name: Validate compressed candidate size
-        if: inputs.push_digests
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          DIGEST='${{ steps.docker_build.outputs.digest }}'
-          [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
-          MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${DIGEST}")"
-          # Provenance wraps the real platform image and its attestation in an
-          # OCI index. Measure the image manifest, not the index/attestation.
-          IMAGE_DIGEST="$(MANIFEST="$MANIFEST" PLATFORM='${{ matrix.platform }}' "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); platform_os, platform_arch = os.environ["PLATFORM"].split("/")[:2]; print(next((item["digest"] for item in manifest.get("manifests", []) if item.get("platform", {}).get("os") == platform_os and item.get("platform", {}).get("architecture") == platform_arch), ""))')"
-          if [[ -n "$IMAGE_DIGEST" ]]; then
-            MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${IMAGE_DIGEST}")"
-          fi
-          COMPRESSED_SIZE="$(MANIFEST="$MANIFEST" "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); print(manifest["config"]["size"] + sum(layer["size"] for layer in manifest["layers"]))')"
-          LIMIT=$((1025 * 1024 * 1024))
-          echo "${DIGEST} compressed_size=$((COMPRESSED_SIZE / 1024 / 1024)) MiB limit=1025 MiB"
-          (( COMPRESSED_SIZE < LIMIT ))
-
-      - name: Resolve real Docker CLI test dependencies through abxpkg
-        id: mount_dependencies
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mount_env="$(
-            uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:mount_validation_binaries"
-          )"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$mount_env" >> "$GITHUB_ENV"
-
-          for binary_name in bash chmod curl exportfs mkdir mount.cifs mount.nfs smbd sudo systemctl umount; do
-            binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
-            test -L "$binary"
-            test -x "$binary"
-          done
-
-      - name: Diagnose failed mount dependency installation
-        if: failure() && steps.mount_dependencies.conclusion == 'failure'
-        shell: bash
-        run: |
-          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
-          sudo journalctl --since '-10 minutes' --no-pager \
-            -u smbd -u nmbd -u samba-ad-dc -u nfs-server
-          ps -eo pid,ppid,stat,etimes,wchan:32,comm
-
-      - name: Validate Docker mounts, ports, CLI and compose concurrently
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        env:
-          ARCHIVEBOX_IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: |
-          set -Eeuo pipefail
-          # These checks own separate collections, Compose projects and ports.
-          # Subshells keep mount cleanup traps and environment changes isolated.
-          validate_mounts() (
-            set -Eeuo pipefail
-            NFS_SOURCE="/srv/archivebox-ci-nfs"
-            SMB_SOURCE="/srv/archivebox-ci-smb"
-            NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
-            SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
-            mounted_nfs=0
-            mounted_smb=0
-
-            # Invoked by the subshell EXIT trap.
-            # shellcheck disable=SC2329
-            cleanup() {
-              local original_status=$? cleanup_status=0
-              trap - EXIT
-              if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
-                cleanup_status=1
-              fi
-              if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
-                cleanup_status=1
-              fi
-              if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
-                cleanup_status=1
-              fi
-              if [[ "$original_status" != "0" ]]; then
-                exit "$original_status"
-              fi
-              exit "$cleanup_status"
-            }
-            trap cleanup EXIT
-
-            "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-            "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
-            "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-            "$SUDO_BINARY" "$BASH_BINARY" -c \
-              "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
-            "$SUDO_BINARY" "$BASH_BINARY" -c \
-              "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
-
-            "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
-            "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
-            "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
-            "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
-            "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
-
-            "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
-            mounted_nfs=1
-            "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
-              -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
-            mounted_smb=1
-
-            NFS_TEST_DIR="$NFS_MOUNT" \
-            SMB_TEST_DIR="$SMB_MOUNT" \
-            IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
-              "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
-          )
-          validate_cli() (
-            set -Eeuo pipefail
-            "$MKDIR_BINARY" "${{ github.workspace }}/data"
-            "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
-
-            "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
-            "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
-
-            "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
-            # The argument form above exercises the complete default pipeline.
-            # Keep a real capture for stdin parsing without repeating every plugin.
-            echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add --plugins=wget,hashes
-
-            archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
-            [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
-            [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
-
-            "$DOCKER_BINARY" compose run --rm archivebox init
-            "$DOCKER_BINARY" compose up -d --wait
-            "$DOCKER_BINARY" network connect archivebox_default www-nginx
-            index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-            admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
-            [[ "$index_html" == *ArchiveBox* ]]
-            [[ "$admin_js" == *window.django* ]]
-
-            "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
-            index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-            [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
-            "$DOCKER_BINARY" compose down
-          )
-          validate_ports() (
-            IMAGE="$ARCHIVEBOX_IMAGE" "$BASH_BINARY" bin/validate_docker_ports.sh
-          )
-
-          checks=(mounts ports cli)
-          pids=()
-          validate_mounts > "$RUNNER_TEMP/docker-mounts.log" 2>&1 &
-          pids+=("$!")
-          validate_ports > "$RUNNER_TEMP/docker-ports.log" 2>&1 &
-          pids+=("$!")
-          validate_cli > "$RUNNER_TEMP/docker-cli.log" 2>&1 &
-          pids+=("$!")
-          status=0
-          for index in "${!checks[@]}"; do
-            check="${checks[$index]}"
-            echo "::group::Docker $check validation"
-            if wait "${pids[$index]}"; then
-              echo "PASS: Docker $check validation"
-            else
-              echo "::error::Docker $check validation failed"
-              status=1
-            fi
-            cat "$RUNNER_TEMP/docker-$check.log"
-            echo "::endgroup::"
-          done
-          exit "$status"
-
-      - name: Validate Docker port compatibility
-        if: ${{ !(inputs.full_tests && matrix.platform == 'linux/amd64') }}
-        shell: bash
-        env:
-          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
-
-      - name: Validate compressed image size
-        if: inputs.full_tests && !inputs.push_digests
-        env:
-          BUILD_METADATA: ${{ steps.docker_build_pr.outputs.metadata }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          LOCAL_IMAGE_ID="$("$DOCKER_BINARY" image inspect '${{ matrix.local_tag }}' --format '{{.Id}}')"
-          "$UV_BINARY" run --no-cache --no-project python - \
-            '${{ runner.temp }}/archivebox-image.tar' \
-            '${{ steps.docker_build_pr.outputs.digest }}' \
-            "$LOCAL_IMAGE_ID" \
-            "$((1025 * 1024 * 1024))" <<'PY'
-          from hashlib import sha256
-          import json
-          import os
-          import sys
-          import tarfile
-
-          archive_path, pushed_digest, local_image_id, limit_text = sys.argv[1:]
-          limit = int(limit_text)
-          metadata = json.loads(os.environ["BUILD_METADATA"])
-          if metadata["containerimage.digest"] != pushed_digest:
-              raise SystemExit("Build push transaction digest does not match the action output")
-          if not pushed_digest.startswith("sha256:") or len(pushed_digest) != 71:
-              raise SystemExit(f"Invalid pushed digest: {pushed_digest}")
-
-          with tarfile.open(archive_path) as archive:
-              def read_blob(descriptor):
-                  algorithm, digest = descriptor["digest"].split(":", 1)
-                  if algorithm != "sha256":
-                      raise SystemExit(f"Unsupported OCI digest: {descriptor['digest']}")
-                  member = archive.extractfile(f"blobs/sha256/{digest}")
-                  if member is None:
-                      raise SystemExit(f"Missing OCI blob: {descriptor['digest']}")
-                  blob = member.read()
-                  if len(blob) != descriptor["size"]:
-                      raise SystemExit(f"OCI size mismatch: {descriptor['digest']}")
-                  if sha256(blob).hexdigest() != digest:
-                      raise SystemExit(f"OCI digest mismatch: {descriptor['digest']}")
-                  return blob
-
-              index_member = archive.extractfile("index.json")
-              if index_member is None:
-                  raise SystemExit("OCI export is missing index.json")
-              index = json.load(index_member)
-
-              manifests = []
-
-              def collect(descriptor):
-                  document = json.loads(read_blob(descriptor))
-                  media_type = descriptor["mediaType"]
-                  if "image.index" in media_type or "manifest.list" in media_type:
-                      for child in document["manifests"]:
-                          collect(child)
-                  elif "image.manifest" in media_type:
-                      config_type = document["config"]["mediaType"]
-                      if "image.config" in config_type or "container.image" in config_type:
-                          manifests.append(document)
-
-              for root in index["manifests"]:
-                  collect(root)
-              if len(manifests) != 1:
-                  raise SystemExit(f"Expected one runnable image manifest, found {len(manifests)}")
-
-              manifest = manifests[0]
-              config_digest = manifest["config"]["digest"]
-              if metadata["containerimage.config.digest"] != config_digest:
-                  raise SystemExit("OCI config digest does not match the pushed build result")
-              if local_image_id != config_digest:
-                  raise SystemExit("Locally tested image does not match the OCI build result")
-              descriptors = [manifest["config"], *manifest["layers"]]
-              for descriptor in descriptors:
-                  read_blob(descriptor)
-              total = sum(descriptor["size"] for descriptor in descriptors)
-
-          print(
-              f"{pushed_digest} compressed_size={total // 1024 // 1024} MiB "
-              f"limit={limit // 1024 // 1024} MiB",
-          )
-          if total > limit:
-              raise SystemExit(f"{pushed_digest} is over the compressed size limit")
-          PY
-
       - name: Export digest
-        if: inputs.push_digests
-        env:
-          ABX_DL_IMAGE_REF: ${{ steps.abx_dl_image.outputs.image }}
-          ARTIFACT_NAME: ${{ matrix.artifact_name }}
         shell: bash
         run: |
           set -Eeuo pipefail
           mkdir -p /tmp/digests
           digest="${{ steps.docker_build.outputs.digest }}"
           touch "/tmp/digests/${digest#sha256:}"
-          printf '%s\n' '${{ inputs.source_ref || github.sha }}' > /tmp/digests/source-ref
-          printf '%s\n' "$ABX_DL_IMAGE_REF" > "/tmp/digests/abx-dl-image-$ARTIFACT_NAME"
 
       - name: Upload digest
-        if: inputs.push_digests
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.artifact_name }}
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1
+
+  publish:
+    name: publish multiarch tags
+    runs-on: ubuntu-24.04
+    needs:
+      - build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: latest
+          install: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digest-*
+          merge-multiple: true
+
+      - name: Collect Docker tags
+        id: docker_meta
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(python3 - <<'PY'
+          import tomllib
+
+          with open("pyproject.toml", "rb") as f:
+              print(tomllib.load(f)["project"]["version"])
+          PY
+          )"
+          BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
+          SHORT_SHA="${GITHUB_SHA::12}"
+          test -n "$BRANCH_TAG"
+          test -n "$SHORT_SHA"
+
+          {
+            echo 'dockerhub_tags<<EOF'
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${DOCKERHUB_IMAGE}:latest"
+            fi
+            echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
+            echo "${DOCKERHUB_IMAGE}:${VERSION}"
+            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+            echo 'EOF'
+            echo 'ghcr_tags<<EOF'
+            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+              echo "${GHCR_IMAGE}:latest"
+            fi
+            echo "${GHCR_IMAGE}:${BRANCH_TAG}"
+            echo "${GHCR_IMAGE}:${VERSION}"
+            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+            echo 'EOF'
+            echo "version=${VERSION}"
+            echo "branch_tag=${BRANCH_TAG}"
+            echo "short_sha=${SHORT_SHA}"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "[+] Publishing Docker Hub tags:"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
+          fi
+          printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+          echo "[+] Publishing GHCR tags:"
+          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
+            printf '%s\n' "${GHCR_IMAGE}:latest"
+          fi
+          printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+
+      - name: Create Docker Hub manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
+          [[ "${#DIGESTS[@]}" -gt 0 ]]
+
+          TAG_ARGS=()
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do
+            REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}")
+          done
+
+          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Create GHCR manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
+          [[ "${#DIGESTS[@]}" -gt 0 ]]
+
+          TAG_ARGS=()
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
+
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do
+            REFS+=("${GHCR_IMAGE}@sha256:${digest}")
+          done
+
+          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Inspect published images
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && docker buildx imagetools inspect "$tag"
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
+
+      - name: Update README
+        uses: peter-evans/dockerhub-description@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: archivebox/archivebox

.github/workflows/docs-links.yml

@@ -1,38 +0,0 @@
-name: Documentation link checks
-
-on:
-  schedule:
-    - cron: '17 10 * * *' # Daily at 10:17 UTC, away from the busy top of the hour.
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-env:
-  PYTHONIOENCODING: utf-8
-
-jobs:
-  links:
-    name: Documentation links and anchors
-    runs-on: ubuntu-24.04
-    timeout-minutes: 5
-    permissions:
-      contents: read
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-      - name: List tracked documentation and source inputs
-        run: |
-          git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > "$RUNNER_TEMP/link-inputs.txt"
-      - uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
-        with:
-          lycheeVersion: v0.24.2
-          args: >-
-            --config .github/lychee.toml
-            --root-dir "${{ github.workspace }}"
-            --files-from "${{ runner.temp }}/link-inputs.txt"
-          fail: true
-          failIfEmpty: true

.github/workflows/docs.yml

@@ -1,195 +0,0 @@
-name: Documentation tests
-
-on:
-  schedule:
-    - cron: '7 5,17 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-env:
-  PYTHONIOENCODING: utf-8
-  USE_COLOR: "False"
-  SHOW_PROGRESS: "False"
-
-jobs:
-  changes:
-    runs-on: ubuntu-24.04
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      key: ${{ steps.check.outputs.key }}
-    steps:
-      - id: check
-        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
-
-  discover:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    name: Discover documentation examples
-    runs-on: ubuntu-24.04
-    timeout-minutes: 5
-    outputs:
-      matrix: ${{ steps.inventory.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-      - name: Install ArchiveBox development dependencies
-        run: uv sync --no-cache --locked --dev
-      - name: Resolve documentation tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
-        run: |
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          uv run --no-cache --no-sync abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries"
-          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
-          mmdc_binary="$ABXPKG_LIB_DIR/env/bin/mmdc"
-          nginx_binary="$ABXPKG_LIB_DIR/env/bin/nginx"
-          test -L "$bash_binary"
-          test -L "$mmdc_binary"
-          test -L "$nginx_binary"
-          test -x "$bash_binary"
-          test -x "$mmdc_binary"
-          test -x "$nginx_binary"
-          {
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-            echo "BASH_BINARY=$bash_binary"
-          } >> "$GITHUB_ENV"
-      - id: inventory
-        name: Validate the complete authored documentation inventory
-        run: |
-          matrix="$(uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py matrix)"
-          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
-
-  examples:
-    name: docs/${{ matrix.name }}
-    needs: discover
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 15
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        with:
-          python-version: "3.13"
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-      - name: Install ArchiveBox
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        run: uv sync --no-cache --locked --dev
-      - name: Resolve documentation runtime through abxpkg
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
-        run: |
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          dependency_sources=(
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
-          )
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
-          fi
-          if [[ '${{ matrix.name }}' == ubuntu-AGENTS || '${{ matrix.name }}' == ubuntu-skills-archivebox-SKILL || '${{ matrix.name }}' == ubuntu-docs-Configuration || '${{ matrix.name }}' == ubuntu-docs-Setting-up-Authentication ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          fi
-          if [[ '${{ matrix.environment }}' == docker ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
-          elif [[ '${{ matrix.environment }}' == macos ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")
-          fi
-          uv run --no-cache --no-sync abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            "${dependency_sources[@]}"
-          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
-          python_binary="$ABXPKG_LIB_DIR/env/bin/python"
-          uv_binary="$ABXPKG_LIB_DIR/env/bin/uv"
-          test -L "$bash_binary"
-          test -x "$bash_binary"
-          test -L "$python_binary"
-          test -x "$python_binary"
-          test -L "$uv_binary"
-          test -x "$uv_binary"
-          {
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-            echo "BASH_BINARY=$bash_binary"
-            echo "PYTHON_BINARY=$python_binary"
-            echo "UV_BINARY=$uv_binary"
-          } >> "$GITHUB_ENV"
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            sudo_binary="$ABXPKG_LIB_DIR/env/bin/sudo"
-            test -L "$sudo_binary"
-            test -x "$sudo_binary"
-            echo "SUDO_BINARY=$sudo_binary" >> "$GITHUB_ENV"
-          fi
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-          fi
-          if [[ '${{ matrix.environment }}' == docker ]]; then
-            docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
-            test -L "$docker_binary"
-            test -x "$docker_binary"
-            echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV"
-          elif [[ '${{ matrix.environment }}' == macos ]]; then
-            test -L "$ABXPKG_LIB_DIR/env/bin/brew"
-            test -x "$ABXPKG_LIB_DIR/env/bin/brew"
-          fi
-      - name: Build the exact checkout's documentation image
-        if: ${{ matrix.environment == 'docker' }}
-        run: |
-          ABX_DL_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(dep.removeprefix("abx-dl==") for dep in tomllib.load(open("pyproject.toml", "rb"))["project"]["dependencies"] if dep.startswith("abx-dl==")))')"
-          "$DOCKER_BINARY" build --build-arg "ABX_DL_IMAGE=archivebox/abx-dl:${ABX_DL_VERSION}" --tag archivebox/archivebox:dev .
-      - name: Run the Ubuntu, Docker, or macOS documentation lane
-        if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
-        run: uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
-      - name: Run the root documentation lane
-        if: ${{ matrix.environment == 'root' }}
-        run: |
-          "$SUDO_BINARY" \
-            --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
-            "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
-      - name: Diagnose failed Linux package installation
-        if: failure() && runner.os == 'Linux'
-        run: |
-          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
-          apt-config dump Acquire::Retries Acquire::http::Timeout Acquire::https::Timeout
-          ps -eo pid,ppid,stat,etimes,wchan:32,comm
-
-  complete:
-    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
-    needs: [changes, discover, examples]
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - run: echo 'Scheduled work completed successfully.'

.github/workflows/duplicate-issue-detection.yml

@@ -0,0 +1,59 @@
+name: Duplicate Issue Detection
+
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  check-duplicates:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Install opencode
+        run: curl -fsSL https://opencode.ai/install | bash
+
+      - name: Check for duplicate issues
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENCODE_PERMISSION: |
+            {
+              "bash": {
+                "gh issue*": "allow",
+                "gh pr*": "allow",
+                "*": "deny"
+              },
+              "webfetch": "allow"
+            }
+        run: |
+          opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created:
+
+          Issue number:
+          ${{ github.event.issue.number }}
+
+          Lookup this issue and search through existing issues and PRs (excluding #${{ github.event.issue.number }}) in this repository to find any potential duplicates of this new issue.
+          Consider:
+          1. Similar titles or descriptions
+          2. Same error messages or symptoms
+          3. Related functionality or components
+          4. Similar feature requests
+
+          If you find any potential duplicates, please comment on the new issue with:
+          - A brief explanation of why it might be a duplicate
+          - Links to the potentially duplicate issues or PRs
+          - A suggestion to check those issues first
+
+          Use this format for the comment:
+          'This issue might be a duplicate of existing issues. Please check:
+          - #[issue_number]: [brief description of similarity]
+
+          Feel free to ignore if none of these address your specific case.'
+
+          If no clear duplicates are found, do not comment."

.github/workflows/lint.yml

@@ -1,38 +1,51 @@
 name: Run linters
 
 on:
-  workflow_call:
+  workflow_dispatch:
+  push:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   UV_NO_SOURCES: "1"
+  PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
 
 jobs:
   lint:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
       - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/setup-python@v5
         with:
           python-version: "3.13"
           architecture: x64
 
       - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        uses: astral-sh/setup-uv@v6
         with:
-          version: "0.11.3"
+          version: "0.10.6"
           enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Verify checked dependency lock
-        run: uv lock --no-cache --check
 
       - name: Install dependencies with uv
-        run: uv sync --no-cache --locked --dev --extra sonic --extra debug
+        run: |
+          uv venv
+          uv pip install --all-groups -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[sonic,debug]"
 
       - name: Run prek
-        run: uv run --no-cache --no-sync --no-sources prek run --all-files
+        run: uv run --no-sync --no-sources prek run --all-files

.github/workflows/macos.yml

@@ -1,48 +0,0 @@
-name: macOS compatibility
-
-on:
-  schedule:
-    - cron: '51 2,10,18 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-jobs:
-  changes:
-    runs-on: ubuntu-24.04
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      key: ${{ steps.check.outputs.key }}
-    steps:
-      - id: check
-        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
-
-  install-cli:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    uses: ./.github/workflows/test.yml
-    with:
-      platform: macos
-    secrets: inherit
-
-  python-artifacts:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    uses: ./.github/workflows/pip.yml
-    with:
-      platforms: '["macos-15"]'
-    secrets: inherit
-
-  complete:
-    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
-    needs: [changes, install-cli, python-artifacts]
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - run: echo 'Scheduled macOS coverage completed successfully.'

.github/workflows/pip.yml

@@ -1,248 +1,109 @@
 name: Build Pip package
 
 on:
+  workflow_dispatch:
   workflow_call:
-    inputs:
-      platforms:
-        description: Runner matrix for package compatibility
-        required: false
-        default: '["ubuntu-24.04"]'
-        type: string
-      full_tests:
-        description: Run the full package compatibility matrix instead of one release smoke check
-        required: false
-        default: true
-        type: boolean
+  push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
-  UV_VERSION: "0.11.3"
+  PYTHON_VERSION: "3.13"
 
 jobs:
   build:
-    name: build distributions
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+    permissions:
+      id-token: write
 
-      - name: Set up Python 3.13
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+    runs-on: ubuntu-24.04
+    environment: pypi
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
         with:
-          python-version: "3.13"
+          version: "0.10.6"
+          enable-cache: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
           architecture: x64
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+      - name: APT install archivebox dev + run dependencies
+        uses: awalsh128/cache-apt-pkgs-action@latest
         with:
-          version: ${{ env.UV_VERSION }}
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
 
-      - name: Build every distribution
+      - name: Wait for released ArchiveBox deps on PyPI
         run: |
-          set -Eeuo pipefail
-          uv build --no-cache --all-packages --no-sources --out-dir dist --clear
-
-          shopt -s nullglob
-          artifacts=(dist/*)
-          wheels=(dist/archivebox-*.whl)
-          sdists=(dist/archivebox-*.tar.gz)
-          [[ "${#artifacts[@]}" -gt 0 ]]
-          [[ "${#wheels[@]}" -eq 1 ]]
-          [[ "${#sdists[@]}" -eq 1 ]]
-          printf '%s\n' "$GITHUB_SHA" > dist/COMMIT_SHA
-          uv run --no-cache --no-project python - <<'PY'
-          import hashlib
-          from pathlib import Path
-
-          root = Path("dist")
-          wheels = list(root.glob("archivebox-*.whl"))
-          sdists = list(root.glob("archivebox-*.tar.gz"))
-          if len(wheels) != 1 or len(sdists) != 1:
-              raise SystemExit("Expected exactly one ArchiveBox wheel and one sdist")
-          artifacts = [root / "COMMIT_SHA", wheels[0], sdists[0]]
-          checksum_lines = []
-          for artifact in artifacts:
-              digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
-              line = f"{digest}  {artifact.name}"
-              checksum_lines.append(line)
-              print(line)
-          Path("dist/SHA256SUMS").write_text("\n".join(checksum_lines) + "\n")
-          PY
-
-      - name: Release wheel import and CLI smoke
-        if: ${{ !inputs.full_tests }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          shopt -s nullglob
-          wheels=(dist/archivebox-*.whl)
-          [[ "${#wheels[@]}" -eq 1 ]]
-          uv sync --locked --no-cache --no-sources --no-dev --no-install-project
-          uv pip install --no-cache --no-deps --python .venv/bin/python "${wheels[0]}"
-          unset PYTHONPATH
-          cd "$(mktemp -d)"
-          VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project python -c 'import archivebox'
-          VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project archivebox version
-
-      - name: Upload every distribution
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: python-distributions
-          path: dist/*
-          if-no-files-found: error
-          retention-days: 7
-
-  install-smoke:
-    name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }}
-    if: inputs.full_tests
-    needs: build
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ${{ fromJSON(inputs.platforms) }}
-        python: ["3.13", "3.14"]
-        artifact: [wheel, sdist]
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: ${{ env.UV_VERSION }}
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Download distributions
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: python-distributions
-          path: ${{ runner.temp }}/python-distributions
-
-      - name: Install and smoke-test the built ${{ matrix.artifact }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          shopt -s nullglob
-
-          case "${{ matrix.artifact }}" in
-            wheel) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.whl) ;;
-            sdist) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.tar.gz) ;;
-            *) exit 2 ;;
-          esac
-          [[ "${#candidates[@]}" -eq 1 ]]
-          artifact="${candidates[0]}"
-
-          smoke_root="$(mktemp -d "$RUNNER_TEMP/archivebox-package-smoke.XXXXXX")"
-          trap 'rm -rf "$smoke_root"' EXIT
-          smoke_env="$smoke_root/env"
-          data_dir="$smoke_root/data"
-          mkdir -p "$data_dir"
-
-          dependency_artifacts=()
-          while IFS= read -r dependency_artifact; do
-            dependency_artifacts+=("$dependency_artifact")
-          done < <(uv run --no-cache --no-project python - <<'PY'
+          python - <<'PY'
+          import json
+          import re
+          import sys
+          import time
           import tomllib
+          import urllib.request
 
-          locked = {
-              package["name"]: package
-              for package in tomllib.load(open("uv.lock", "rb"))["package"]
-          }
-          for package_name in ("abx-dl", "abx-plugins", "abxpkg", "abxbus"):
-              package = locked[package_name]
-              wheels = package.get("wheels") or []
-              if len(wheels) != 1:
-                  raise SystemExit(f"Expected exactly one locked wheel for {package_name}")
-              print(wheels[0]["url"])
+          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
+          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
+          required = {}
+          for dep in deps:
+              for name in watched:
+                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
+                  if match:
+                      required[name] = match.group(2)
+
+          deadline = time.monotonic() + 300
+          missing = required.copy()
+          while missing and time.monotonic() < deadline:
+              for name, version in list(missing.items()):
+                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
+                      releases = json.load(resp)["releases"]
+                  if version in releases:
+                      print(f"{name} {version} is available on PyPI")
+                      missing.pop(name)
+                  else:
+                      print(f"{name} {version} is not available on PyPI yet")
+              if missing:
+                  time.sleep(10)
+
+          if missing:
+              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
+              sys.exit(1)
           PY
-          )
 
-          uv venv --python "${{ matrix.python }}" "$smoke_env"
-          uv pip install --no-cache \
-            --python "$smoke_env" \
-            "${dependency_artifacts[@]}" \
-            "$artifact"
+      - name: UV install archivebox dev + run sub-dependencies
+        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
 
-          unset PYTHONPATH
-          cd "$data_dir"
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync python - <<'PY'
-          import os
-          from pathlib import Path
+      - name: UV build archivebox and archivebox/pkgs/* packages
+        run: |
+          uv build --all
 
-          import archivebox
+      - name: Publish new package wheels and sdists to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # ignore when publish to PyPI fails due to duplicate tag
+        continue-on-error: true
+        with:
+          password: ${{ secrets.PYPI_PAT_SECRET }}
 
-          installed_path = Path(archivebox.__file__).resolve()
-          environment_path = Path(os.environ["VIRTUAL_ENV"]).resolve()
-          workspace_path = Path(os.environ["GITHUB_WORKSPACE"]).resolve()
-          assert installed_path.is_relative_to(environment_path), (installed_path, environment_path)
-          assert not installed_path.is_relative_to(workspace_path), (installed_path, workspace_path)
-          print(installed_path)
-          PY
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox version
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox init
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox status
+      - name: UV install archivebox and archivebox/pkgs/* locally for tests
+        run: uv sync --all-extras --no-sources --no-cache
 
-          if [[ "${{ runner.os }}" == "Linux" ]]; then
-            root_tool_dir="/root/archivebox-package-smoke-${{ matrix.artifact }}-${{ matrix.python }}"
-            uv_binary="$(command -v uv)"
-            sudo rm -rf "$root_tool_dir"
-            sudo mkdir -p "$root_tool_dir/data"
-
-            root_install_args=()
-            for dependency_artifact in "${dependency_artifacts[@]}"; do
-              root_install_args+=(--with "$dependency_artifact")
-            done
-            sudo UV_TOOL_DIR="$root_tool_dir/tools" UV_TOOL_BIN_DIR="$root_tool_dir/bin" \
-              "$uv_binary" tool install --no-cache --python "${{ matrix.python }}" \
-              "${root_install_args[@]}" "$artifact"
-
-            sudo useradd --system --create-home --home-dir /var/lib/archivebox --shell /bin/bash archivebox 2>/dev/null || true
-            sudo rm -rf /var/lib/archivebox
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data' && archivebox init"
-            sudo stat -c %U /var/lib/archivebox | grep -qx archivebox
-            sudo stat -c %U "$root_tool_dir/data" | grep -qx archivebox
-            if sudo find "$root_tool_dir/data" -xdev -user root -print -quit | grep -q .; then
-              echo "Root-owned files remain in $root_tool_dir/data" >&2
-              exit 1
-            fi
-
-            sudo mkdir "$root_tool_dir/external"
-            sudo ln -s "$root_tool_dir/external" "$root_tool_dir/data/external-link"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data' && archivebox status"
-            sudo stat -c %U "$root_tool_dir/external" | grep -qx root
-
-            sudo mkdir "$root_tool_dir/data-universal-init"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-universal-init' && archivebox status --init"
-            sudo stat -c %U "$root_tool_dir/data-universal-init" | grep -qx archivebox
-
-            sudo mkdir "$root_tool_dir/data-install"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-install' && archivebox install --dry-run"
-            sudo stat -c %U "$root_tool_dir/data-install" | grep -qx archivebox
-
-            sudo useradd --create-home archivebox-smoke-user
-            smoke_user_group="$(id -gn archivebox-smoke-user)"
-            sudo mkdir "$root_tool_dir/data-mixed-owner"
-            sudo chown archivebox-smoke-user:root "$root_tool_dir/data-mixed-owner"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-mixed-owner' && archivebox init"
-            sudo stat -c %U:%G "$root_tool_dir/data-mixed-owner" | grep -qx "archivebox-smoke-user:$smoke_user_group"
-            if sudo find "$root_tool_dir/data-mixed-owner" -xdev -user root -print -quit | grep -q .; then
-              echo "Root-owned files remain in $root_tool_dir/data-mixed-owner" >&2
-              exit 1
-            fi
-            sudo rm -rf "$root_tool_dir"
-          fi
+      - name: UV run archivebox init + archivebox version
+        run: |
+          mkdir -p data && cd data
+          uv run --no-sync --no-sources archivebox init \
+          && uv run --no-sync --no-sources archivebox version
+          # && uv run archivebox add 'https://example.com' \
+          # && uv run archivebox status \
+          # || (echo "UV Failed to run archivebox!" && exit 1)

.github/workflows/prepare-release.yml

@@ -1,55 +0,0 @@
-name: Prepare release version
-
-on:
-  workflow_call:
-    outputs:
-      run_jobs:
-        description: Whether this revision owns its release version and should run expensive jobs
-        value: ${{ jobs.prepare.outputs.run_jobs }}
-
-permissions:
-  contents: write
-
-jobs:
-  prepare:
-    name: Prepare the version before starting expensive jobs
-    permissions:
-      contents: write
-    runs-on: ubuntu-24.04
-    outputs:
-      run_jobs: ${{ github.event_name != 'push' || steps.channel.outputs.stable == 'true' || steps.version.outputs.action == 'ready' || steps.version.outputs.action == 'reserved' }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          token: ${{ github.event_name == 'push' && secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - name: Keep synchronized stable branches on the released version
-        id: channel
-        if: github.event_name == 'push'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
-          git fetch --no-tags origin main
-          MAIN_SHA="$(git rev-parse origin/main)"
-          # Stable reconciliation may preserve dev history with a merge commit.
-          # Its tree still matches main; bumping it would undo synchronization
-          # before the stable publisher can advance the Docker :dev tag.
-          if [[ "$GITHUB_REF_NAME" == dev && "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
-             git merge-base --is-ancestor "$MAIN_SHA" "$GITHUB_SHA" &&
-             git diff --quiet "$MAIN_SHA" "$GITHUB_SHA"; then
-            echo 'stable=true' >> "$GITHUB_OUTPUT"
-          else
-            echo 'stable=false' >> "$GITHUB_OUTPUT"
-          fi
-
-      - id: version
-        if: github.event_name == 'push' && steps.channel.outputs.stable != 'true'
-        uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
-        with:
-          package: archivebox
-          branch: ${{ github.ref_name }}
-          tag-prefix: v
-          version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}

.github/workflows/release-runner.yml

@@ -0,0 +1,50 @@
+name: Release State
+
+on:
+  push:
+    branches:
+      - '**'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  id-token: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  release-state:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+          ref: ${{ github.ref_name }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.10.6"
+          enable-cache: false
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Configure git identity
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Run release script
+        env:
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+          GH_TOKEN: ${{ github.token }}
+          PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
+        run: ./bin/release.sh

.github/workflows/release.yml

@@ -1,595 +1,41 @@
 name: Release
 
-on:
-  workflow_run:
-    workflows: [CI]
-    types: [completed]
-    branches: [dev, main]
-  workflow_dispatch:
-    inputs:
-      release_branch:
-        description: Branch being promoted
-        required: true
-        type: choice
-        options: [main, dev]
-      release_sha:
-        description: Exact tested commit to release
-        required: true
-        type: string
-      ci_run_id:
-        description: Successful CI run containing the tested Python artifacts
-        required: true
-        type: string
-      docker_digest_run_id:
-        description: Manual Docker digest build run containing the tested image digests
-        required: true
-        type: string
+# Orchestrates the full release pipeline:
+#   1. Build and publish pip package to PyPI
+#   2. Build Homebrew formula after pip
+#   3. Build Docker images in parallel
+#
+# Individual workflows also run on push for CI (see their own triggers).
+# Manual fallback for the hosted release pipeline. The normal prerelease loop
+# is run by bin/release.sh locally so published GitHub releases do not launch a
+# duplicate release job.
 
-permissions: {}
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  packages: write
+  id-token: write
 
 concurrency:
-  group: release-${{ github.repository }}-${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
-  cancel-in-progress: false
-
-env:
-  DOCKERHUB_IMAGE: archivebox/archivebox
-  GHCR_IMAGE: ghcr.io/archivebox/archivebox
-  RELEASE_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
-  RELEASE_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.release_sha || github.event.workflow_run.head_sha }}
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  candidate:
-    if: >-
-      github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.event == 'push' &&
-      (github.event.workflow_run.head_branch == 'dev' || github.event.workflow_run.head_branch == 'main') ||
-      (github.event_name == 'workflow_dispatch' && inputs.release_branch == 'main')
-    runs-on: ubuntu-24.04
-    permissions:
-      actions: read
-      contents: read
-    outputs:
-      ready: ${{ steps.verified.outputs.ready }}
-      artifact_run_id: ${{ steps.verified.outputs.artifact_run_id }}
-      digest_run_id: ${{ steps.verified.outputs.digest_run_id }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          ref: ${{ env.RELEASE_SHA }}
-      - id: owner
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          BRANCH_SHA="$(git ls-remote origin "refs/heads/$RELEASE_BRANCH" | cut -f1)"
-          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && "$BRANCH_SHA" != "$RELEASE_SHA" ]]; then
-            echo 'ready=false' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
-              echo "Refusing to publish non-stable version $VERSION as a main release" >&2
-              exit 1
-            }
-            git fetch --no-tags origin main
-            git merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
-              echo "Refusing to publish $RELEASE_SHA: it is not on main" >&2
-              exit 1
-            }
-            TAG_TARGET="$(git ls-remote origin "refs/tags/v$VERSION" | cut -f1)"
-            if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && -n "$TAG_TARGET" && "$TAG_TARGET" != "$RELEASE_SHA" ]]; then
-              echo 'ready=false' >> "$GITHUB_OUTPUT"
-              exit 0
-            fi
-            echo 'ready=true' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          # A stable release fast-forwards dev to main. Only main publishes it;
-          # the resulting dev workflow is not a new release candidate.
-          if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]]; then
-            echo 'ready=false' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          TARGET="$(git ls-remote origin "refs/tags/release-candidate/$VERSION" | awk '{print $1}')"
-          [[ "$TARGET" == "$RELEASE_SHA" ]] && echo 'ready=true' >> "$GITHUB_OUTPUT" || echo 'ready=false' >> "$GITHUB_OUTPUT"
+  pip:
+    name: Publish to PyPI
+    uses: ./.github/workflows/pip.yml
+    secrets: inherit
 
-      - name: Require successful full CI for this commit
-        id: verified
-        if: steps.owner.outputs.ready == 'true'
-        env:
-          GH_TOKEN: ${{ github.token }}
-          MANUAL_CI_RUN_ID: ${{ inputs.ci_run_id }}
-          MANUAL_DIGEST_RUN_ID: ${{ inputs.docker_digest_run_id }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then
-            CI_RUN_ID="$MANUAL_CI_RUN_ID"
-            DIGEST_RUN_ID="$MANUAL_DIGEST_RUN_ID"
-            CI_STATUS="$(gh run view "$CI_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status,conclusion,headSha,event,name \
-              --jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
-            [[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
-          else
-            CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
-              --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
-              --json databaseId,status,conclusion \
-              --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
-            [[ -n "$CI_RUN_ID" ]] || exit 0
-            DIGEST_RUN_ID="$CI_RUN_ID"
-          fi
-          ARTIFACT_RUN_ID="$CI_RUN_ID"
-          {
-            echo 'ready=true'
-            echo "artifact_run_id=$ARTIFACT_RUN_ID"
-            echo "digest_run_id=$DIGEST_RUN_ID"
-          } >> "$GITHUB_OUTPUT"
+  homebrew:
+    name: Update Homebrew formula
+    needs: pip
+    uses: ./.github/workflows/homebrew.yml
+    secrets: inherit
 
-  python-release:
-    name: Publish tested Python artifacts
-    needs: candidate
-    if: needs.candidate.outputs.ready == 'true'
-    runs-on: ubuntu-24.04
-    env:
-      CI_RUN_ID: ${{ needs.candidate.outputs.artifact_run_id }}
-      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
-    environment: pypi
-    permissions:
-      actions: read
-      contents: write
-      deployments: read
-      id-token: write
-    outputs:
-      version: ${{ steps.version.outputs.version }}
-      tag: ${{ steps.version.outputs.tag }}
-      release_ready: ${{ steps.release-state.outputs.ready }}
-    steps:
-      - name: Checkout the tested commit
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          ref: ${{ env.RELEASE_SHA }}
-          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      # This precedes every stable upload, including workflow_dispatch. A green
-      # build is not evidence that our automatically updated test servers ran it.
-      - name: Download tested Docker base-image metadata
-        if: env.RELEASE_BRANCH == 'main'
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          path: ${{ runner.temp }}/tested-docker-artifacts
-          pattern: digest-*
-          merge-multiple: true
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
-
-      - name: Require matching acceptance on Cabbage
-        if: env.RELEASE_BRANCH == 'main'
-        env:
-          GH_TOKEN: ${{ github.token }}
-          STAGING_DOCKER_ARTIFACT_DIR: ${{ runner.temp }}/tested-docker-artifacts
-        run: uv run --no-project python bin/verify-staging-release.py
-
-      - name: Resolve release binaries through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" >/dev/null
-          for name in uv gh git jq curl; do
-            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
-            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
-          done
-          {
-            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
-            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
-            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-            echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
-          } >> "$GITHUB_ENV"
-
-      - name: Download exact tested distributions
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: python-distributions
-          path: ${{ runner.temp }}/python-distributions
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.CI_RUN_ID }}
-
-      - name: Record release version
-        id: version
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$($UV_BINARY run --no-cache --no-project python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
-          echo "tag=v$VERSION" >> "$GITHUB_OUTPUT"
-
-      - name: Validate release branch and version
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION='${{ steps.version.outputs.version }}'
-          case "$RELEASE_BRANCH" in
-            main)
-              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
-                echo "Refusing to publish non-stable version $VERSION from main" >&2
-                exit 1
-              }
-              ;;
-            dev)
-              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]] || {
-                echo "Refusing to publish ArchiveBox $VERSION from dev; only archivebox:main may publish non-prerelease versions." >&2
-                exit 1
-              }
-              ;;
-            *)
-              echo "Refusing to publish ArchiveBox $VERSION from unsupported branch $RELEASE_BRANCH; only archivebox:dev and archivebox:main may publish." >&2
-              exit 1
-              ;;
-          esac
-
-      - name: Publish the exact tested distributions
-        env:
-          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-          RELEASE_DISTRIBUTIONS_DIR: ${{ runner.temp }}/python-distributions
-        run: ./bin/release.sh
-
-      - name: Confirm this tested commit owns the release tag
-        id: release-state
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          TARGET=""
-          while read -r sha _; do
-            TARGET="$sha"
-          done < <("$GIT_BINARY" ls-remote origin "refs/tags/${{ steps.version.outputs.tag }}")
-          if [[ "$TARGET" == "$RELEASE_SHA" ]]; then
-            echo "ready=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "ready=false" >> "$GITHUB_OUTPUT"
-          fi
-
-  docker-release:
-    name: Publish tested multiarch image
-    if: >-
-      (github.event_name == 'workflow_dispatch') ||
-      (github.event.workflow_run.conclusion == 'success' &&
-       github.event.workflow_run.event == 'push')
-    needs: [candidate, python-release]
-    runs-on: ubuntu-24.04
-    env:
-      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
-    permissions:
-      actions: read
-      contents: write
-      packages: write
-    steps:
-      - name: Checkout the tested commit
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          ref: ${{ env.RELEASE_SHA }}
-          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - name: Reconcile dev to the tested stable release
-        if: env.RELEASE_BRANCH == 'main' && needs.python-release.outputs.release_ready == 'true'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          git fetch --no-tags origin main dev
-          MAIN_SHA="$(git rev-parse origin/main)"
-          DEV_SHA="$(git rev-parse origin/dev)"
-          if [[ "$MAIN_SHA" != "$RELEASE_SHA" || "$DEV_SHA" == "$RELEASE_SHA" ]]; then
-            exit 0
-          fi
-          push_dev() {
-            if ! git push origin "$1:refs/heads/dev"; then
-              CURRENT_DEV="$(git ls-remote origin refs/heads/dev | cut -f1)"
-              if [[ "$CURRENT_DEV" == "$DEV_SHA" ]]; then
-                echo "Failed to push dev although it still points to $DEV_SHA" >&2
-                exit 1
-              fi
-              echo "Leaving dev at $CURRENT_DEV: it advanced during release reconciliation"
-            fi
-          }
-          if git merge-base --is-ancestor "$DEV_SHA" "$RELEASE_SHA"; then
-            push_dev "$RELEASE_SHA"
-            exit 0
-          else
-            STATUS=$?
-            [[ "$STATUS" -eq 1 ]] || exit "$STATUS"
-          fi
-          BASE_SHA="$(git merge-base "$MAIN_SHA" "$DEV_SHA")"
-          if git diff --quiet "$BASE_SHA" "$DEV_SHA" -- . ':!pyproject.toml' ':!uv.lock' ':!etc/package.json'; then
-            :
-          else
-            STATUS=$?
-            [[ "$STATUS" -eq 1 ]] || exit "$STATUS"
-            echo "Leaving dev at $DEV_SHA: it contains changes beyond release metadata"
-            exit 0
-          fi
-          normalized_metadata() {
-            case "$2" in
-              pyproject.toml)
-                git show "$1:$2" | awk '
-                  /^\[/ {section = $0}
-                  section == "[project]" && /^version = "[^"]+"$/ {print "version = \"__VERSION__\""; next}
-                  section == "[tool.bumpver]" && /^current_version = "[^"]+"$/ {print "current_version = \"__VERSION__\""; next}
-                  {print}
-                ' ;;
-              uv.lock)
-                git show "$1:$2" | awk 'previous == "name = \"archivebox\"" && /^version = / {print "version = \"__VERSION__\""; previous = $0; next} {print; previous = $0}' ;;
-              etc/package.json)
-                git show "$1:$2" | sed -E 's/^(  "version": )"[^"]+"(,?)$/\1"__VERSION__"\2/' ;;
-            esac
-          }
-          for path in pyproject.toml uv.lock etc/package.json; do
-            BASE_METADATA="$(normalized_metadata "$BASE_SHA" "$path")"
-            DEV_METADATA="$(normalized_metadata "$DEV_SHA" "$path")"
-            if [[ "$BASE_METADATA" != "$DEV_METADATA" ]]; then
-              echo "Leaving dev at $DEV_SHA: $path contains changes beyond release version metadata"
-              exit 0
-            fi
-          done
-          # A concurrent release bot may bump only dev's own version metadata.
-          # Keep that commit as a parent while preserving the exact tested stable tree.
-          git switch --detach "$MAIN_SHA"
-          git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \
-            merge -s ours --no-ff -m "Merge tested stable release into dev" "$DEV_SHA"
-          MERGE_SHA="$(git rev-parse HEAD)"
-          push_dev "$MERGE_SHA"
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      - name: Resolve release and Docker binaries through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
-          for name in uv gh git jq docker; do
-            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
-            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
-          done
-          {
-            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
-            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
-            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-            echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
-          } >> "$GITHUB_ENV"
-
-      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
-        with:
-          # These images were already built and tested; only publish manifests.
-          driver: docker
-
-      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ github.token }}
-
-      - name: Download exact tested image digests
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          path: /tmp/digests
-          pattern: digest-*
-          merge-multiple: true
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
-
-      - name: Verify exact successful digest build run
-        shell: bash
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          set -Eeuo pipefail
-          RUN_STATUS="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status --jq '.status')"
-          RUN_CONCLUSION="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json conclusion --jq '.conclusion')"
-          [[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]]
-          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
-            RUN_SHA="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha')"
-            [[ "$RUN_SHA" == "$RELEASE_SHA" ]]
-          fi
-
-      - name: Verify digest set and collect tags
-        id: docker_meta
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          [[ "${#DIGESTS[@]}" -eq 2 ]]
-          for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
-          [[ "$(< /tmp/digests/source-ref)" == "$RELEASE_SHA" ]]
-
-          VERSION='${{ needs.python-release.outputs.version }}'
-          SHORT_SHA="${RELEASE_SHA::12}"
-          TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}^{}")"
-          TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
-          if [[ -z "$TAG_TARGET" ]]; then
-            TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
-            TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
-          fi
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
-              echo "Refusing to tag :latest with prerelease $VERSION" >&2
-              exit 1
-            }
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] || {
-              echo "Refusing to tag :latest: v$VERSION does not point to $RELEASE_SHA" >&2
-              exit 1
-            }
-            $GIT_BINARY fetch --no-tags origin main
-            $GIT_BINARY merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
-              echo "Refusing to tag :latest: $RELEASE_SHA is not on main" >&2
-              exit 1
-            }
-            MAIN_VERSION="$($GIT_BINARY show origin/main:pyproject.toml | sed -nE 's/^version = "([^"]+)".*/\1/p' | head -n 1)"
-            [[ "$MAIN_VERSION" == "$VERSION" ]] || {
-              echo "Refusing to tag :latest: main is now $MAIN_VERSION, not $VERSION" >&2
-              exit 1
-            }
-          fi
-          SYNC_DEV=false
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            $GIT_BINARY fetch --no-tags origin main dev
-            MAIN_TARGET="$($GIT_BINARY rev-parse origin/main)"
-            DEV_TARGET="$($GIT_BINARY rev-parse origin/dev)"
-            # Recovery can add CI/docs fixes after the immutable release tag.
-            # Both branches and every shipped application file must agree.
-            if $GIT_BINARY merge-base --is-ancestor "$MAIN_TARGET" "$DEV_TARGET" &&
-               $GIT_BINARY diff --quiet "$MAIN_TARGET" "$DEV_TARGET" &&
-               $GIT_BINARY diff --quiet "$RELEASE_SHA" "$MAIN_TARGET" -- . ':!.github' ':!README.md'; then
-              SYNC_DEV=true
-            fi
-          fi
-          {
-            echo 'dockerhub_tags<<EOF'
-            if [[ "$RELEASE_BRANCH" == dev ]]; then
-              echo "${DOCKERHUB_IMAGE}:dev"
-            elif [[ "$RELEASE_BRANCH" == main ]]; then
-              echo "${DOCKERHUB_IMAGE}:latest"
-              [[ "$SYNC_DEV" == true ]] && echo "${DOCKERHUB_IMAGE}:dev"
-            fi
-            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
-            echo 'EOF'
-            echo 'ghcr_tags<<EOF'
-            if [[ "$RELEASE_BRANCH" == dev ]]; then
-              echo "${GHCR_IMAGE}:dev"
-            elif [[ "$RELEASE_BRANCH" == main ]]; then
-              echo "${GHCR_IMAGE}:latest"
-              [[ "$SYNC_DEV" == true ]] && echo "${GHCR_IMAGE}:dev"
-            fi
-            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
-            echo 'EOF'
-          } >> "$GITHUB_OUTPUT"
-
-      - name: Create Docker Hub manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          TAG_ARGS=()
-          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
-          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Create GHCR manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          TAG_ARGS=()
-          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
-          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Verify published tags contain the exact tested images
-        env:
-          DOCKERHUB_TAGS: ${{ steps.docker_meta.outputs.dockerhub_tags }}
-          GHCR_TAGS: ${{ steps.docker_meta.outputs.ghcr_tags }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          # Both architectures already passed CLI/version/capture acceptance by
-          # digest. Verify promotion preserves those exact images and attestations
-          # instead of downloading and running the same image again on one arch.
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
-          EXPECTED="$("$DOCKER_BINARY" buildx imagetools create --dry-run "${REFS[@]}" | "$JQ_BINARY" -ceS '.manifests | sort_by(.digest)')"
-          "$JQ_BINARY" -e 'map(select(.platform.os == "linux") | .platform.architecture) | sort == ["amd64", "arm64"]' <<< "$EXPECTED"
-          while IFS= read -r image; do
-            [[ -n "$image" ]] || continue
-            ACTUAL="$("$DOCKER_BINARY" buildx imagetools inspect --raw "$image" | "$JQ_BINARY" -ceS 'select(.schemaVersion == 2) | .manifests | sort_by(.digest)')"
-            [[ "$ACTUAL" == "$EXPECTED" ]] || { echo "Published tag differs from tested images: $image" >&2; exit 1; }
-            echo "Verified exact tested images: $image"
-          done < <(printf '%s\n' "$DOCKERHUB_TAGS" "$GHCR_TAGS")
-
-      - name: Record published image source
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          # workflow_run can report the default branch's head even when this
-          # run published a different branch/SHA. Preserve the verified target.
-          [[ "$RELEASE_BRANCH" == dev || "$RELEASE_BRANCH" == main ]]
-          [[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]
-          mkdir -p "$RUNNER_TEMP/published-image-source"
-          # shellcheck disable=SC2016
-          "$JQ_BINARY" -n --arg branch "$RELEASE_BRANCH" --arg sha "$RELEASE_SHA" \
-            '{branch: $branch, sha: $sha}' > "$RUNNER_TEMP/published-image-source/source.json"
-
-      - name: Upload published image source
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: published-image-source
-          path: ${{ runner.temp }}/published-image-source/source.json
-          if-no-files-found: error
-
-      - name: Update Docker Hub README
-        uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: archivebox/archivebox
-
-  cascade:
-    name: Cascade verified release
-    if: needs.python-release.outputs.release_ready == 'true'
-    needs:
-      - candidate
-      - python-release
-      - docker-release
-    permissions:
-      actions: read
-      contents: read
-    uses: ArchiveBox/monorepo/.github/workflows/cascade-release.yml@main
-    with:
-      upstream_package: archivebox
-      upstream_version: ${{ needs.python-release.outputs.version }}
-      upstream_artifact_name: python-distributions
-      upstream_run_id: ${{ needs.candidate.outputs.artifact_run_id }}
-    secrets:
-      release_token: ${{ secrets.RELEASE_GH_TOKEN }}
+  docker:
+    name: Build Docker images
+    needs: pip
+    uses: ./.github/workflows/docker.yml
+    secrets: inherit

.github/workflows/screenshots.yml

@@ -1,81 +0,0 @@
-name: Capture publicsite screenshots
-
-on:
-  schedule:
-    - cron: '37 5,17 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-jobs:
-  changes:
-    runs-on: ubuntu-24.04
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      key: ${{ steps.check.outputs.key }}
-    steps:
-      - id: check
-        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
-
-  deploy:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Setup Python and uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: false
-
-      - name: Install ArchiveBox and screenshot dependencies
-        run: uv sync --no-cache --frozen
-
-      - name: Build UI screenshot gallery
-        run: bin/collect_ui_screenshots.sh
-        env:
-          BASE_URL: http://archivebox.localhost:5797
-
-      - name: Verify gallery was generated from this exact revision
-        run: |
-          uv run --no-cache --no-sync python - <<'PY'
-          import json
-          import os
-          import re
-          import tomllib
-          from pathlib import Path
-
-          provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
-          version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
-          assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
-          assert provenance["version"] == version, provenance
-          assert provenance["capture_count"] >= 3 * 35, provenance
-          assert len(provenance["files"]) == provenance["capture_count"], provenance
-          assert all(not re.match(r"^\d+-", name) for name in provenance["files"]), provenance
-          PY
-
-      - name: Save validated screenshots
-        uses: actions/upload-artifact@v4
-        with:
-          name: site-screenshots
-          path: publicsite/screenshots
-          if-no-files-found: error
-          retention-days: 90
-
-  complete:
-    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
-    needs: [changes, deploy]
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - run: echo 'Scheduled work completed successfully.'

.github/workflows/staging-acceptance.yml

@@ -1,117 +0,0 @@
-name: Staging acceptance
-
-on:
-  workflow_run:
-    workflows: [Release]
-    types: [completed]
-
-permissions: {}
-
-concurrency:
-  group: staging-acceptance
-  cancel-in-progress: false
-
-jobs:
-  published:
-    if: github.event.workflow_run.conclusion == 'success'
-    runs-on: ubuntu-24.04
-    permissions:
-      actions: read
-    outputs:
-      ready: ${{ steps.source.outputs.ready }}
-      sha: ${{ steps.source.outputs.sha }}
-    steps:
-      - id: image
-        env:
-          GH_TOKEN: ${{ github.token }}
-          RELEASE_RUN_ID: ${{ github.event.workflow_run.id }}
-        run: |
-          set -Eeuo pipefail
-          published=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RELEASE_RUN_ID/jobs?per_page=100" --jq 'any(.jobs[]; .name == "Publish tested multiarch image" and .conclusion == "success")')
-          echo "published=$published" >> "$GITHUB_OUTPUT"
-      - name: Download published image source
-        if: steps.image.outputs.published == 'true'
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: published-image-source
-          path: ${{ runner.temp }}/published-image-source
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ github.event.workflow_run.id }}
-      - name: Select published dev image
-        id: source
-        if: steps.image.outputs.published == 'true'
-        run: |
-          set -Eeuo pipefail
-          # The triggering run may report the default branch as its head;
-          # this artifact names the image source verified by the publisher.
-          source="$RUNNER_TEMP/published-image-source/source.json"
-          jq -e 'type == "object" and (keys | sort) == ["branch", "sha"]' "$source" >/dev/null
-          branch=$(jq -er '.branch | select(type == "string")' "$source")
-          sha=$(jq -er '.sha | select(type == "string")' "$source")
-          [[ "$branch" == dev || "$branch" == main ]]
-          [[ "$sha" =~ ^[0-9a-f]{40}$ ]]
-          echo "sha=$sha" >> "$GITHUB_OUTPUT"
-          if [[ "$branch" == dev ]]; then
-            echo 'ready=true' >> "$GITHUB_OUTPUT"
-          else
-            echo 'ready=false' >> "$GITHUB_OUTPUT"
-          fi
-  acceptance:
-    # The privileged key is never exposed to PRs or arbitrary dispatch refs.
-    needs: published
-    if: needs.published.outputs.ready == 'true'
-    runs-on: [self-hosted, Linux, X64, ugnas-linux]
-    environment: staging
-    timeout-minutes: 110
-    permissions:
-      contents: read
-      deployments: write
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [cabbage]
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
-        with:
-          ref: ${{ needs.published.outputs.sha }}
-          fetch-depth: 0
-          persist-credentials: false
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e
-        with:
-          version: '0.11.3'
-      - name: Prepare browser and restricted staging identity
-        env:
-          STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
-          STAGING_KNOWN_HOSTS: ${{ secrets.STAGING_KNOWN_HOSTS }}
-          ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/staging-browser
-        run: |
-          set -Eeuo pipefail
-          umask 077
-          printf '%s\n' "$STAGING_SSH_KEY" > "$RUNNER_TEMP/staging-key"
-          printf '%s\n' "$STAGING_KNOWN_HOSTS" > "$RUNNER_TEMP/staging-known-hosts"
-          # Use the candidate's normal provider/cache path, not a second
-          # Playwright-managed browser that bypasses abxpkg validation.
-          ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
-          CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
-          uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
-          CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
-          test -x "$CHROME_BINARY"
-          echo "CHROME_BINARY=$CHROME_BINARY" >> "$GITHUB_ENV"
-      - name: Verify actual deployment, full captures and rendered snapshots
-        env:
-          GH_TOKEN: ${{ github.token }}
-          STAGING_TARGET: ${{ matrix.target }}
-          STAGING_SHA: ${{ needs.published.outputs.sha }}
-          STAGING_KEY_FILE: ${{ runner.temp }}/staging-key
-          STAGING_KNOWN_HOSTS_FILE: ${{ runner.temp }}/staging-known-hosts
-        run: uv run --no-project --with playwright==1.63.0 python bin/check-staging-deployment.py
-      - name: Remove staging identity
-        if: always()
-        run: rm -f "$RUNNER_TEMP/staging-key" "$RUNNER_TEMP/staging-known-hosts"
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
-        if: always()
-        with:
-          name: staging-${{ matrix.target }}-${{ needs.published.outputs.sha }}
-          path: staging-evidence/
-          if-no-files-found: error

.github/workflows/test-parallel.yml

@@ -2,254 +2,368 @@ name: Parallel Tests
 
 on:
   workflow_dispatch:
-    inputs:
-      test_file:
-        description: Run one discovered test file
-        required: true
-        default: archivebox/tests/test_cli_list.py
-        type: string
-  workflow_call:
-    inputs:
-      test_file:
-        description: Run one discovered test file; empty runs the complete matrix
-        required: false
-        default: ""
-        type: string
+  pull_request:
+    branches: [dev, main, master]
+  push:
+    branches: [dev]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
   UV_NO_SOURCES: "1"
-  CI_PYTHON_VERSION: "3.13"
 
 jobs:
   discover-tests:
-    name: Discover every test
-    # Selection must run on GitHub so an offline NAS cannot block the fallback.
+    name: Discover test files
     runs-on: ubuntu-24.04
     outputs:
-      tests: ${{ steps.discover.outputs.tests }}
-      runner: ${{ steps.runner.outputs.use-runner || '["ubuntu-24.04"]' }}
+      test-files: ${{ steps.set-matrix.outputs.test-files }}
+      plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}
 
     steps:
-      - name: Select available CI capacity
-        id: runner
-        if: vars.UGNAS_CI_ENABLED == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
-        uses: ArchiveBox/monorepo/.github/actions/select-ci-runner@main
-        with:
-          github-token: ${{ secrets.RUNNER_STATUS_TOKEN }}
-
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: ${{ env.CI_PYTHON_VERSION }}
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Resolve discovery tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+      - name: Discover test files
+        id: set-matrix
         run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          test -n "$ABXPKG_SPEC"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PYTHON_BINARY="$pythonLocation/bin/python"
-          test -x "$PYTHON_BINARY"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
-          {
-            echo "PYTHON_BINARY=$ABXPKG_LIB_DIR/env/bin/python"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-          } >> "$GITHUB_ENV"
-          test -L "$ABXPKG_LIB_DIR/env/bin/python"
-          test -x "$ABXPKG_LIB_DIR/env/bin/python"
-          test -L "$ABXPKG_LIB_DIR/env/bin/jq"
-          test -x "$ABXPKG_LIB_DIR/env/bin/jq"
+          # Find all main test files
+          all_tests=$(find archivebox/tests -maxdepth 1 -name "test_*.py" -type f | sort)
 
-      - name: Discover every test exactly once
-        id: discover
-        env:
-          TEST_FILE: ${{ inputs.test_file }}
-          UGNAS_CI_MAX_JOBS: ${{ vars.UGNAS_CI_MAX_JOBS || 3 }}
+          # Create JSON array with test file info
+          json_array="["
+          first=true
+          for test_file in $all_tests; do
+            if [ "$first" = true ]; then
+              first=false
+            else
+              json_array+=","
+            fi
+
+            # Extract a display name for the test
+            name="main/$(basename $test_file .py | sed 's/^test_//')"
+            needs_chromium=false
+            if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then
+              needs_chromium=true
+            fi
+            needs_sonic=false
+            if grep -Eiq "shutil\\.which\\([\"']sonic|SEARCH_BACKEND_ENGINE=.*sonic|worker_sonic" "$test_file"; then
+              needs_sonic=true
+            fi
+
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
+          done
+          json_array+="]"
+
+          echo "test-files=$json_array" >> $GITHUB_OUTPUT
+          echo "Found $(echo $all_tests | wc -w) test files"
+          echo "$json_array" | jq '.'
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Discover plugin tests
+        id: set-plugin-matrix
         run: |
-          set -Eeuo pipefail
-          mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
-          test "${#output[@]}" -eq 2
-          if [[ -n "$TEST_FILE" ]]; then
-            output[1]="$("$JQ_BINARY" -ce '[.[] | select(.paths == [env.TEST_FILE])] | select(length == 1)' <<< "${output[1]}")"
-          fi
-          echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
-          echo "${output[0]}"
-          "$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"
+          plugin_tests=$(find abx-plugins/abx_plugins/plugins -maxdepth 2 -type d -name tests | sed 's#abx-plugins/abx_plugins/plugins/##; s#/tests##' | sort)
 
-  tests:
+          json_array="["
+          first=true
+          for plugin_name in $plugin_tests; do
+            if [ "$first" = true ]; then
+              first=false
+            else
+              json_array+=","
+            fi
+
+            needs_chromium=false
+            if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
+              needs_chromium=true
+            fi
+            needs_sonic=false
+            if [ "$plugin_name" = "search_backend_sonic" ]; then
+              needs_sonic=true
+            fi
+
+            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium,\"needs_sonic\":$needs_sonic}"
+          done
+          json_array+="]"
+
+          echo "plugin-tests=$json_array" >> $GITHUB_OUTPUT
+          echo "Found $(echo $plugin_tests | wc -w) plugin test suites"
+          echo "$json_array" | jq '.'
+
+  run-tests:
     name: ${{ matrix.test.name }}
+    runs-on: ubuntu-24.04
     needs: discover-tests
-    # Opt in trusted branches only; PRs always use GitHub-hosted runners.
-    # Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
-    # The hosted discovery job falls back when workers are offline or the
-    # availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
-    runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
     env:
+      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
       CHROME_HEADLESS: "true"
+      CHROME_SANDBOX: "false"
+      PERSONAS_DIR: /tmp/abx-personas
+      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
 
     strategy:
       fail-fast: false
       matrix:
-        test: ${{ fromJson(needs.discover-tests.outputs.tests) }}
+        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
+        python: ["3.13"]
 
     steps:
-      - name: Set job-local runtime paths
-        env:
-          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
-        run: |
-          workers=2
-          cache_suffix=''
-          if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
-            workers="$(nproc)"
-            cache_suffix="-workers-$workers"
-            # Avoid the hosted image's cold Chromium executable page faults.
-            # abxpkg owns installation and reuse in the existing runtime cache.
-            echo "CHROME_BINPROVIDERS=playwright,puppeteer" >> "$GITHUB_ENV"
-          fi
-          {
-            echo "CI_TEST_WORKERS=$workers"
-            echo "ABXPKG_CACHE_WORKERS_SUFFIX=$cache_suffix"
-            echo "ABXPKG_LIB_DIR=$RUNNER_TEMP/archivebox-lib"
-            echo "DATA_DIR=$RUNNER_TEMP/archivebox-test-data"
-            echo "PERSONAS_DIR=$RUNNER_TEMP/abx-personas"
-            echo "CHROME_USER_DATA_DIR=$RUNNER_TEMP/abx-personas/Default/chrome_profile"
-          } >> "$GITHUB_ENV"
-
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.CI_PYTHON_VERSION }}
+          python-version: ${{ matrix.python }}
           architecture: x64
-        id: python
 
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
         with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          version: "0.10.6"
 
-      # Self-hosted tool storage survives jobs. Normal abxpkg validation and
-      # install locks allow reuse across files; cold-install tests use tmp_path.
-      - name: Reuse persistent runner tools
-        if: runner.environment == 'self-hosted'
-        env:
-          PERSISTENT_ABXPKG_LIB: ${{ runner.tool_cache }}/abxpkg/${{ github.repository }}/${{ steps.python.outputs.python-version }}
-        run: |
-          mkdir -p "$PERSISTENT_ABXPKG_LIB"
-          echo "ABXPKG_LIB_DIR=$PERSISTENT_ABXPKG_LIB" >> "$GITHUB_ENV"
-
-      # Restored binaries still pass normal abxpkg resolution/version checks.
-      # Installation and isolation tests explicitly choose fresh temporary libs.
-      - name: Identify runtime dependency cache
-        shell: bash
-        run: |
-          uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
-          import hashlib
-          import json
-          import tomllib
-          from pathlib import Path
-
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
-          print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
-          PYTHON
-
-      - name: Reuse validated runtime dependencies
-        if: runner.environment != 'self-hosted'
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
         with:
-          path: ${{ runner.temp }}/archivebox-lib
-          key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}${{ env.ABXPKG_CACHE_WORKERS_SUFFIX }}
+          node-version: 22
+
+      - name: Cache uv
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
           restore-keys: |
-            abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-
+            ${{ runner.os }}-${{ matrix.python }}-uv-
 
-      - name: Resolve optional build dependencies through abxpkg
-        if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
-        env:
-          TEST_EXTRA: ${{ matrix.test.extra }}
-          TEST_PATHS: ${{ matrix.test.paths_arg }}
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          test -n "$ABXPKG_SPEC"
-          dependency_sources=()
-          if [[ -n "$TEST_EXTRA" ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          fi
-          if [[ "$TEST_PATHS" == *test_postgres_backend* ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:postgres_binaries")
-          fi
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-            --install \
-            --no-cache \
-            --json \
-            --lib="$ABXPKG_LIB_DIR" \
-            "${dependency_sources[@]}"
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.1
 
-      - name: Install test dependencies
-        env:
-          TEST_EXTRA: ${{ matrix.test.extra }}
+      - name: Install dependencies with uv
         run: |
-          set -Eeuo pipefail
-          extra_args=()
-          if [[ -n "$TEST_EXTRA" ]]; then
-            extra_args=(--extra "$TEST_EXTRA")
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
+
+      - name: Install Chrome through ArchiveBox
+        if: ${{ matrix.test.needs_chromium }}
+        env:
+          MIN_CHROMIUM_MAJOR: "149"
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
+            exit 1
           fi
-          uv sync --no-cache --locked --dev "${extra_args[@]}"
-      - name: Run ${{ matrix.test.name }}
+          version="$("$candidate" --version || true)"
+          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+          case "$major" in
+            ''|*[!0-9]*) major=0 ;;
+          esac
+          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
+            exit 1
+          fi
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
+          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.test.needs_sonic }}
         env:
-          TEST_PATHS: ${{ matrix.test.paths_arg }}
-          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
         run: |
-          set -Eeuo pipefail
-          echo "Running pytest with $CI_TEST_WORKERS workers on $CI_RUNNER_ENVIRONMENT"
-          read -r -a test_paths <<< "$TEST_PATHS"
-          test_args=()
-          for test_path in "${test_paths[@]}"; do
-            test -f "$test_path"
-            test_args+=("$GITHUB_WORKSPACE/$test_path")
-          done
-          # Each test still gets isolated DB/filesystem state from conftest.
-          # Keep one worker pool per batch instead of starting it for every file.
-          mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest"
-          uv run --no-cache --no-sync --no-sources pytest -vs -n "$CI_TEST_WORKERS" "${test_args[@]}" --durations=20 \
-            --basetemp="$RUNNER_TEMP/pytest" \
-            --junitxml="$RUNNER_TEMP/test-results/results.xml"
-      - name: Save test results
-        if: always()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
+
+      - name: Run test - ${{ matrix.test.name }}
+        run: |
+          mkdir -p tests/out
+          set +e
+          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
+          status=$?
+          set -e
+          if [ "$status" -eq 5 ]; then
+            echo "No tests collected from ${{ matrix.test.path }}; treating as an empty test module."
+            exit 0
+          fi
+          exit "$status"
+
+  plugin-tests:
+    name: ${{ matrix.plugin.name }}
+    runs-on: ubuntu-24.04
+    needs: discover-tests
+    env:
+      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
+      CHROME_HEADLESS: "true"
+      CHROME_SANDBOX: "false"
+      PERSONAS_DIR: /tmp/abx-personas
+      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
+
+    strategy:
+      fail-fast: false
+      matrix:
+        plugin: ${{ fromJson(needs.discover-tests.outputs.plugin-tests) }}
+
+    steps:
+      - uses: actions/checkout@v4
         with:
-          name: test-results-${{ strategy.job-index }}
-          path: ${{ runner.temp }}/test-results/*.xml
-          if-no-files-found: error
-          retention-days: 14
-      - name: Save failed runtime diagnostics
-        if: failure()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+          submodules: true
+          fetch-depth: 1
+
+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v5
         with:
-          name: runtime-logs-${{ strategy.job-index }}
-          path: ${{ runner.temp }}/pytest/**/*.log
-          if-no-files-found: ignore
-          retention-days: 14
+          python-version: "3.13"
+          architecture: x64
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.10.6"
+
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Cache uv
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-3.13-uv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.1
+
+      - name: Install dependencies with uv
+        run: |
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
+
+      - name: Install Chrome through ArchiveBox
+        if: ${{ matrix.plugin.needs_chromium }}
+        env:
+          MIN_CHROMIUM_MAJOR: "149"
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+          case "$major" in
+            ''|*[!0-9]*) major=0 ;;
+          esac
+          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
+            exit 1
+          fi
+          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
+          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+
+      - name: Install Sonic through ArchiveBox
+        if: ${{ matrix.plugin.needs_sonic }}
+        env:
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_INSTALL_TIMEOUT: "900"
+          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install search_backend_sonic
+
+          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="sonic", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
+          if [ ! -x "$candidate" ]; then
+            echo "ArchiveBox did not install an executable Sonic binary: ${candidate:-not found}" >&2
+            exit 1
+          fi
+          version="$("$candidate" --version || true)"
+          echo "Using ArchiveBox-installed Sonic: $candidate (${version:-version unknown})"
+          echo "SONIC_BINARY=$candidate" >> "$GITHUB_ENV"
+          echo "$(dirname "$candidate")" >> "$GITHUB_PATH"
+
+      - name: Run plugin tests - ${{ matrix.plugin.name }}
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
+        run: |
+          uv run --no-sync --no-sources bash ./bin/test_plugins.sh "${{ matrix.plugin.plugin }}" --no-coverage

.github/workflows/test.yml

@@ -1,60 +1,156 @@
-name: Install and CLI Compatibility
+name: Run tests
+on: [push]
 
-on:
-  workflow_call:
-    inputs:
-      platform:
-        description: Platform for install and CLI compatibility
-        required: false
-        default: linux
-        type: string
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   DOCKER_IMAGE: archivebox-ci
-  ARCHIVEBOX_IMAGE: archivebox-ci
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
   UV_NO_SOURCES: "1"
 
 jobs:
-  install_cli_platform:
-    name: CLI install / ${{ matrix.os_name }} / Python ${{ matrix.python }}
+  python_tests:
     runs-on: ${{ matrix.os }}
+    env:
+      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
 
     strategy:
-      fail-fast: false
       matrix:
-        include: ${{ fromJSON(inputs.platform == 'macos' && '[{"os":"macos-15","os_name":"macOS","python":"3.13"},{"os":"macos-15","os_name":"macOS","python":"3.14"}]' || '[{"os":"ubuntu-24.04","os_name":"linux","python":"3.14"}]') }}
+        os: [ubuntu-22.04]
+        # os: [ubuntu-22.04, macos-latest, windows-latest]
+        python: ["3.13"]
 
     steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
-      ### Set up Python
+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
+      ### Setup Python & JS Languages
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
+          architecture: x64
 
       - name: Install uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: astral-sh/setup-uv@v4
         with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          version: "0.10.6"
+
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      ### Install Python & JS Dependencies
+      - name: Cache uv
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python }}-uv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
 
       - name: Install dependencies with uv
-        run: uv sync --no-cache --locked --dev
-
-      - name: ArchiveBox full install check
         run: |
-          set -Eeuo pipefail
-          DATA_DIR="$PWD/tests/out/data"
-          mkdir -p "$DATA_DIR"
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox init
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox install
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox version
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox status
+          uv venv
+          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
+
+      ### Run the tests
+      - name: Directory listing for debugging
+        run: |
+          pwd
+          ls
+
+      - name: Archivebox version
+        run: |
+          mkdir -p tests/out/data
+          DATA_DIR="$PWD/tests/out/data" uv run --no-sync --no-sources archivebox version
+
+      - name: Test built package with pytest
+        # TODO: remove this exception for windows once we get tests passing on that platform
+        if: ${{ !contains(matrix.os, 'windows') }}
+        run: |
+          mkdir -p tests/out
+          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs
+
+      - name: Run plugin tests
+        if: ${{ !contains(matrix.os, 'windows') }}
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
+        run: |
+          uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage
+
+  docker_tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
+      # - uses: satackey/action-docker-layer-caching@v0.0.8
+
+      - name: Build image
+        run: |
+          docker build . -t "$DOCKER_IMAGE"
+
+      - name: Init data dir
+        run: |
+          mkdir "${{ github.workspace }}/data"
+          docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init
+
+      - name: Run test server
+        run: |
+          sudo bash -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
+          docker run --name www-nginx -p 80:80 -d nginx
+
+      - name: Add link
+        run: |
+          docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local
+
+      - name: Add stdin link
+        run: |
+          echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add
+
+      - name: List links
+        run: |
+          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
+          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }
+
+      - name: Start docker-compose stack
+        run: |
+          docker-compose run archivebox init
+          docker-compose up -d
+          sleep 5
+          curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
+          curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'
+
+      - name: Check added urls show up in index
+        run: |
+          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
+          curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
+          docker-compose down || true

.github/workflows/update-homebrew-tap.yml

@@ -0,0 +1,35 @@
+name: Update Homebrew tap
+
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: update-homebrew-tap-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dispatch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Dispatch tap update
+        env:
+          HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
+        run: |
+          if [ -z "$HOMEBREW_TAP_TOKEN" ]; then
+            echo "HOMEBREW_TAP_TOKEN is not configured; tap hourly fallback will update the formula."
+            exit 0
+          fi
+
+          curl --fail-with-body \
+            -X POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${HOMEBREW_TAP_TOKEN}" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            https://api.github.com/repos/ArchiveBox/homebrew-archivebox/dispatches \
+            -d '{"event_type":"archivebox-dev-updated"}'

.gitignore

@@ -8,8 +8,6 @@ __pycache__/
 .mypy_cache/
 .eggs/
 tests/out/
-/docs/screenshots/
-/publicsite/screenshots/
 
 # Coverage
 .coverage
@@ -68,5 +66,3 @@ ArchiveBox.conf
 
 # Local Claude Code task locks (not project state)
 .claude/scheduled_tasks.lock
-
-/_site/

.pre-commit-config.yaml

@@ -56,8 +56,6 @@ repos:
       - id: mixed-line-ending
       - id: fix-byte-order-marker
       - id: end-of-file-fixer
-        # Preserve the exact bytes of captured cryptographic proof fixtures.
-        exclude: ^archivebox/tests/fixtures/legacy_proofs/.*\.json$
       - id: detect-private-key
       - id: debug-statements
       - id: forbid-submodules

AGENTS.md

@@ -1,83 +1,14 @@
-# ArchiveBox Agent Guide
+I found 31 unique `no mocking` prompts in recent Codex history, across 21 session transcripts. Only one transcript literally started with `no mocking`; most had it later as testing guidance. Consolidated advice:
 
-ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
-
-## Shared Standards
-
-- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
-- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
-- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
-- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
-- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
-- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
-- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
-- Start behavior fixes with a red failing test when a test is requested or practical.
-- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
-- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
-
-## Concurrency Contract
-
-- A collection has one orchestrator at a time. Local PID/process checks may warn about obvious same-machine duplicates, but must not claim to enforce ownership across machines or shared filesystems.
-- SQLite remains supported with concurrent short writes from CLI and server processes. Keep the SQLite database on a local filesystem, never NFS/SMB, and use short autocommit/CAS updates instead of long transactions or database locks.
-- Network calls, hook execution, filesystem migrations, and other long work belong in the orchestrator. Never hold a database transaction or lock across that work.
-- PostgreSQL and shared data directories are the path to future multi-machine scheduling. Coordinate that work through database-backed per-crawl/per-snapshot claims; do not introduce file leases or timer-based orchestrator election.
-
-## Development Setup
-
-```bash
-uv sync --dev --all-extras
-mkdir -p data
-cd data
-uv run --project .. archivebox init --install
-```
-
-Run collection commands from inside an initialized data directory:
-
-```bash
-cd data
-uv run --project .. archivebox status
-uv run --project .. archivebox add --plugins=parse_txt_urls 'https://example.com/'
-uv run --project .. archivebox run
-```
-
-## User-Facing Setup
-
-Recommended CLI install:
-
-```bash
-uv tool install --python 3.13 --prerelease explicit --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
-mkdir -p ~/archivebox/data
-cd ~/archivebox/data
-archivebox init --install
-archivebox add --plugins=parse_txt_urls 'https://example.com/'
-```
-
-Alternative install methods:
-
-- Docker Compose / Docker
-- Homebrew
-- Debian package
-- pip
-
-## Basic Usage
-
-```bash
-cd ~/archivebox/data
-archivebox version
-archivebox help
-archivebox status
-archivebox install
-archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
-archivebox list --json --with-headers
-archivebox search 'example'
-archivebox update --filter-type=domain example.com
-archivebox remove --yes --delete --filter-type=exact 'https://example.com/docs-basic-usage'
-archivebox run
-```
-
-## Verification
-
-Run the test suite with `uv run pytest archivebox/tests -x` and repository checks
-with `uv run prek run --all-files`. For focused work, select the relevant tests.
-
-Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.
+- Tests must hit real user-facing code paths: CLI commands, REST/API calls, browser UI, real hooks, real ArchiveBox data dirs, real pytest fixtures, and real subprocess/binary behavior.
+- No mocking, faking, simulating, monkey patching, handwritten fake objects, fake buses, fake hooks, fake binaries, fake handlers, or direct-post shortcuts when the user path is through UI/extension/CLI.
+- No skipped, xfailed, flaky, or “works around platform” tests. Flakiness is treated as a bug, especially on macOS/browser flows.
+- Prefer live integration tests over narrow unit tests when behavior depends on browsers, binaries, ArchiveBox crawls, plugins, LLMs, or server state.
+- Assertions must validate real correctness: returned values, exit codes, DB rows, filesystem contents, field values, uploaded files, rendered output, and side effects. “No error occurred” or “attribute exists” is not enough.
+- Start fixes with failing red tests that reproduce the missing behavior or regression, then implement the minimal fix and confirm the test passes.
+- Use realistic setup patterns “like a user would”: events + bus + handlers, real browser pages/CDP sessions, real URLs or `pytest-httpserver`, real rows, real snapshots, real installs, real local browser/server state.
+- For ArchiveBox/API tests, use existing `conftest.py` fixtures and test harnesses, real test DB rows/data dirs, and user-facing commands/APIs rather than bespoke helpers.
+- For browser/extension tests, trigger behavior through the real extension UI or actual browser session, not direct posting or mocked browser/session objects.
+- For binary/provider tests, use real binaries and real installs; verify constraints and final installed package metadata, not just install success.
+- For coverage quality, keep tests strict, deterministic, grouped consistently, and use a few larger realistic tests when that gives better surface coverage than many tiny fake unit tests.
+- Avoid weakening test coverage, adding compatibility/shim/fallback layers, or guessing from code shape. Trace root causes, verify assumptions with tests/scripts, and let real type/parse errors surface normally.

Dockerfile

@@ -1,38 +1,8 @@
 # syntax=docker/dockerfile:1.7
 
 # Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# WHY: abx-dl owns Python, Node, Chromium, and extraction dependencies so the
-# standalone downloader stays independent of ArchiveBox's server features.
-# ArchiveBox-only dependencies (server plugins, supervisor, Django) belong in
-# the layers added by this Dockerfile, never in the shared abx-dl base image.
-# REQUIRED INSTALLATION CONTRACT — do not work around failures:
-# 1. IMAGE CONSTRUCTION: preinstall ALL plugins supported by runtime="archivebox"
-#    and ALL their required_binaries, regardless of default enabled state.
-#    Inherit the complete downloader dependencies from the tested abx-dl image;
-#    install every remaining ArchiveBox-only dependency in this image's layers,
-#    including optional server plugins. Never put server dependencies in abx-dl.
-#    Use the normal plugin dependency pipeline: install command -> config.json
-#    required_binaries -> abxpkg. Never add manual apt/pip/npm installs to hide
-#    failures in that pipeline. OS/interpreter bootstrap is a separate concern,
-#    not a second hand-maintained list of plugin dependencies.
-# 2. FINISHED IMAGE: archivebox install and the full plugin installation check
-#    must discover everything already present and be NO-OPS offline as UID 911.
-#    No downloads, package installs, sudo, or root repair may be needed to enable
-#    ANY plugin or start a new crawl. Never run verification as root or manually
-#    preinstall packages before it. If the inherited image lacks a downloader
-#    dependency, fix/release abx-dl and consume that tested image; do not patch
-#    over the missing base dependency here or repair deployed containers by SSH.
-# 3. NON-CONTAINER CI: exercise real runtime auto-installation through the same
-#    CLI/required_binaries/abxpkg path users invoke. GitHub's runner has
-#    passwordless sudo; abxpkg must escalate automatically when needed. Never
-#    manually preinstall dependencies or wrap the tested command in sudo to make
-#    a failure disappear. This is distinct from the finished-image no-op test.
-# 4. SIZE: the complete compressed image budget is 1025 MiB. Preserve ALL
-#    user-facing functionality. Only mechanical/internal
-#    optimizations are allowed. Never remove dependencies, disable features or
-#    plugins, weaken assertions, or skip checks to meet a size limit. If the
-#    complete image exceeds the configured limit, LET CI FAIL and explicitly
-#    ask the maintainer what to do. Do not raise the limit unilaterally.
+# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
+# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
 # Build abx-dl first, then point this file at it:
 #   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
 #       --build-context abxbus=../abxbus \
@@ -40,10 +10,10 @@
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
 #   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:dev \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
 #       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:main
+ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
 
 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
@@ -52,7 +22,6 @@ ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
-ARG ARCHIVEBOX_COMMIT_HASH=""
 
 ENV TZ=UTC \
     LANGUAGE=en_US:en \
@@ -77,12 +46,13 @@ ENV ARCHIVEBOX_USER=archivebox \
     IN_DOCKER=True
 
 ENV CODE_DIR=/app \
-    ABX_RUNTIME=archivebox \
     DATA_DIR=/data \
     CONFIG_DIR=/opt/archivebox \
+    LIB_DIR=/opt/archivebox/lib \
     ABXPKG_LIB_DIR=/opt/archivebox/lib \
     PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
     PERSONAS_DIR=/data/personas \
+    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
     CHROME_HEADLESS=true \
     CHROME_SANDBOX=false \
     CHROME_ISOLATION=crawl \
@@ -96,7 +66,10 @@ ENV TMP_DIR=/tmp/archivebox \
 
 ENV HOME=/home/archivebox \
     XDG_CONFIG_HOME=/home/archivebox/.config \
-    XDG_CACHE_HOME=/opt/archivebox/lib/cache \
+    XDG_CACHE_HOME=/home/archivebox/.cache \
+    ABXPKG_INSTALL_TIMEOUT=600 \
+    ABXPKG_POSTINSTALL_SCRIPTS=True \
+    ABXPKG_MIN_RELEASE_AGE=0 \
     TIMEOUT=600
 
 ENV UV_COMPILE_BYTECODE=false \
@@ -116,93 +89,50 @@ RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
     && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
     && uname -a \
     && sed -n '1,7p' /etc/os-release \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env node \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env uv \
+    && which node && node --version \
+    && which uv && uv self version \
     ) | tee -a /VERSION.txt
 
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends ripgrep \
+    && rm -rf /var/lib/apt/lists/*
+
 FROM archivebox-runtime-base AS archivebox-builder
 
 WORKDIR "$CODE_DIR"
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
     --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
-    <<'EOF'
-echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
-export UV_NO_CACHE=true
-rm -f /etc/apt/apt.conf.d/docker-clean
-apt-get update -qq
-apt-get install -qq -y --no-install-recommends x11-utils
-# Extend abx-dl's existing venv instead of clearing it. Clearing and later
-# copying a complete replacement over the base duplicates every venv byte in
-# the final overlay history even when most packages are unchanged.
-# Do not rewrite inherited native libraries merely to strip them again.
-/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries
-
-# The old Docker-only dependency rewrite predated this checkout's lockfile:
-# it fetched version JSON to avoid stale PyPI Simple listings. That extra API
-# can itself lag or return 503s. The verified lock already records exact public
-# artifact URLs and hashes, including extras and platform markers. Consume it
-# directly, retaining the freshness check instead of resolving a second graph.
-/usr/bin/uv sync \
-    --project /app \
-    --locked \
-    --no-cache \
-    --no-dev \
-    --inexact \
-    --no-install-project \
-    --no-install-workspace \
-    --no-install-package abxbus \
-    --no-install-package abxpkg \
-    --no-install-package abx-plugins \
-    --no-install-package abx-dl \
-    --no-sources
-# Sync may replace inherited dependencies (e.g. pydantic/rich-click), removing
-# their bytecode. Runtime writes are disabled, so missing caches would make
-# every hook compile them again. Compile only missing entries: compileall's
-# timestamp fast path rewrites even valid checked-hash caches, copying the
-# entire inherited venv into this OCI layer for no runtime benefit.
-/usr/bin/uv run --no-project --no-sync python - <<'PY'
-import importlib.util
-from pathlib import Path
-import py_compile
-import sysconfig
-
-for source in Path(sysconfig.get_path("purelib")).rglob("*.py"):
-    cache = Path(importlib.util.cache_from_source(str(source)))
-    if not cache.is_file():
-        py_compile.compile(str(source), doraise=True, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH)
-PY
-builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
-ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
-/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
-while IFS= read -r -d '' native_library; do
-    if grep -Fxq "$native_library" /tmp/archivebox-inherited-libraries; then
-        continue
-    fi
-    magic=''
-    if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
-        "$builder_abxpkg_lib_dir/env/bin/strip" --strip-unneeded "$native_library" || exit $?
-    fi
-done < /tmp/archivebox-native-libraries
-rm -f /tmp/archivebox-native-libraries /tmp/archivebox-inherited-libraries
-rm -f /venv/bin/uv /venv/bin/uvx
-abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils
-abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get autoremove -y
-rm -rf "$builder_abxpkg_lib_dir"
-rm -rf /venv/lib/python3.*/site-packages/pip* \
-    /venv/lib/python3.*/site-packages/wheel* \
-    /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
-rm -rf /var/lib/apt/lists/*
-EOF
+    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
+    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
+    && rm -f /etc/apt/apt.conf.d/docker-clean \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \
+    && /usr/bin/uv pip install setuptools pip wheel \
+    && /usr/bin/uv sync \
+        --refresh \
+        --no-dev \
+        --inexact \
+        --no-install-project \
+        --no-install-workspace \
+        --no-sources \
+    && (find /venv/lib/python3.*/site-packages -type f -name '*.so' -exec strip --strip-unneeded {} + 2>/dev/null || true) \
+    && rm -f /venv/bin/uv /venv/bin/uvx \
+    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-# Compile only ArchiveBox's installed package. Touching all of /venv here would
-# copy every inherited abx-dl file into a new layer merely to change its mtime.
-RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
     && COMMIT_HASH="$( \
-        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-            echo "$ARCHIVEBOX_COMMIT_HASH"; \
-        elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
             HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
             if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
                 echo "$HEAD_REF"; \
@@ -212,19 +142,12 @@ RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
             fi; \
         fi)" \
     && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
-    && /usr/bin/uv pip install --no-cache --no-deps "$CODE_DIR" \
+    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
     && rm -f /venv/bin/uv /venv/bin/uvx \
-    && cd / \
-    && ARCHIVEBOX_PY_DIR="$(/venv/bin/python -c 'import pathlib, archivebox; print(pathlib.Path(archivebox.__file__).parent)')" \
-    && /venv/bin/python -m compileall --invalidation-mode checked-hash -q "$ARCHIVEBOX_PY_DIR" \
-    && test -f "$(/venv/bin/python -c 'import archivebox; print(archivebox.__cached__)')" \
-    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
+    && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
+    && rm -rf "$CODE_DIR/.git"
 
-# The builder installs and purges stripping tools in one layer, so its final
-# filesystem is already runtime-clean. Preserve that ancestry: starting again
-# from archivebox-runtime-base and COPYing /venv would bake a second full venv
-# over the inherited one instead of recording only ArchiveBox's package delta.
-FROM archivebox-builder
+FROM archivebox-runtime-base
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -240,11 +163,12 @@ LABEL name="archivebox" \
 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
 
-# The builder invokes abxpkg as root for temporary ELF stripping, which can
-# rewrite derived state ownership. Restore UID 911 before resolving binaries
-# as the runtime user; the old copy-based final stage hid this dependency.
-RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777,id=archivebox-runtime-config-$TARGETARCH \
-    echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
+COPY --from=archivebox-builder /opt/uv/python /opt/uv/python
+COPY --from=archivebox-builder /venv /venv
+COPY --from=archivebox-builder /app /app
+COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
+
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
     && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
     && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
     && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
@@ -255,60 +179,46 @@ RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777
     && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
     && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
     && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
-    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
-    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
-    && find "$ABXPKG_LIB_DIR" \( ! -user "$DEFAULT_ARCHIVEBOX_UID" -o ! -group "$DEFAULT_ARCHIVEBOX_GID" \) -exec chown -h "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" {} + \
-    && UV_NO_CACHE=true setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
+    && (which sonic && sonic --version) | tee -a /VERSION.txt \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.cache" \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER/.cache/abxbus/semaphores" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \
+    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/abxbus/semaphores" \
     && openssl rand -hex 16 > /etc/machine-id \
     && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-# Install the complete ArchiveBox catalog, including disabled server plugins.
-# Explicit plugin names make installation independent of enabled defaults.
-RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-server-plugins-$TARGETARCH \
-    chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" /var/tmp/abxpkg-cache \
-    && chmod 1777 /var/tmp/abxpkg-cache \
-    && export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
-    && env ABXPKG_TMP_CACHE_DIR=/var/tmp/abxpkg-cache XDG_CACHE_HOME=/var/tmp/abxpkg-cache \
-        setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
-    && UV_NO_CACHE=true /usr/bin/uv run --no-project python -c 'from abx_plugins.plugins.opencode.image import prune_incompatible_image_files; prune_incompatible_image_files()' \
-    # pnpm's explicit store lives under ABXPKG_LIB_DIR, regardless of XDG_CACHE_HOME.
-    # Retaining its download copies made image size depend on whether the build
-    # preserved hardlinks to installed files. Remove the store in this layer;
-    # installed files retain their contents, and the offline check below proves
-    # the complete catalog still works without the disposable package payloads.
-    && rm -rf "$ABXPKG_LIB_DIR/cache/pnpm"
-RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
-    bash -c 'uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_installed; verify_installed()" && abx-dl install $ABX_DOCKER_PLUGINS'
+    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
 
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
     && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && HOME="$TMP_DIR" archivebox init \
+    && archivebox init \
     && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
         "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
         "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
         "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
         "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
         "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
-        "$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
+        "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" "/home/$ARCHIVEBOX_USER/.cache" \
         2>/dev/null || true) \
     && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
 
 RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
 
-RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
-    && test -f "$(/venv/bin/python -c 'import pydantic; print(pydantic.__cached__)')" \
-    && test -f "$(/venv/bin/python -c 'import rich_click; print(rich_click.__cached__)')" \
-    && test -f "$(/venv/bin/python -c 'import platformdirs; print(platformdirs.__cached__)')" \
-    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+RUN "$LIB_DIR/playwright/bin/chromium" --version | tee -a /VERSION.txt \
+    && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
+    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
+    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
+    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
+    && for forbidden_bin in gcc g++ make; do ! command -v "$forbidden_bin" || (echo "Unexpected build tool in runtime: $forbidden_bin=$(command -v "$forbidden_bin")" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
+    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER/.cache" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "/home/$ARCHIVEBOX_USER/.cache/abxbus/semaphores" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "/home/$ARCHIVEBOX_USER/.cache/uv" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install \
     && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
 
 RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
@@ -318,10 +228,10 @@ RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully
 
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
-EXPOSE 5797
+EXPOSE 8000
 
 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:5797/health/' | grep -q 'OK'
+    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
 
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:5797"]
+CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]

archivebox/__init__.py

@@ -38,6 +38,11 @@ ASCII_LOGO = """
 
 PACKAGE_DIR = Path(__file__).resolve().parent
 
+# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models
+# # Migrations reference models like 'machine.Binary' which need to be importable
+# if str(PACKAGE_DIR) not in sys.path:
+#     sys.path.append(str(PACKAGE_DIR))
+
 os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
 os.environ["TZ"] = "UTC"
 
@@ -46,9 +51,10 @@ from .config.permissions import drop_privileges  # noqa
 
 drop_privileges()
 
-from .misc.checks import check_not_root, check_io_encoding  # noqa
+from .misc.checks import check_not_root, check_not_inside_source_dir, check_io_encoding  # noqa
 
 check_not_root()
+check_not_inside_source_dir()
 check_io_encoding()
 
 from .config.version import VERSION  # noqa

archivebox/api/apps.py

@@ -7,11 +7,6 @@ class APIConfig(AppConfig):
     name = "archivebox.api"
     label = "api"
 
-    def ready(self):
-        from archivebox.api.webhooks import register_delete_webhook_field_loader
-
-        register_delete_webhook_field_loader()
-
 
 def register_admin(admin_site):
     from archivebox.api.admin import register_admin

archivebox/api/migrations/0001_initial.py

@@ -12,9 +12,19 @@ import signal_webhooks.fields
 import signal_webhooks.utils
 
 
-# Raw sqlite DDL kept EXACTLY as-is; executed only on sqlite via Django's
-# RunSQL (identical statement splitting → byte-for-byte identical behavior).
-INITIAL_SQL = """
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        ("auth", "0012_alter_user_first_name_max_length"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    sql="""
                 -- Create api_apitoken table
                 CREATE TABLE IF NOT EXISTS api_apitoken (
                     id TEXT PRIMARY KEY NOT NULL,
@@ -60,60 +70,12 @@ INITIAL_SQL = """
                 CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
                 CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
                 CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
-                    """
-
-INITIAL_REVERSE_SQL = """
+                    """,
+                    reverse_sql="""
                 DROP TABLE IF EXISTS api_outboundwebhook;
                 DROP TABLE IF EXISTS api_apitoken;
-                    """
-
-
-def _run_sqlite_only_sql(apps, schema_editor):
-    if schema_editor.connection.vendor != "sqlite":
-        return
-    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_forwards(
-        "api",
-        schema_editor,
-        None,
-        None,
-    )
-
-
-def _run_sqlite_only_sql_reverse(apps, schema_editor):
-    if schema_editor.connection.vendor != "sqlite":
-        return
-    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_backwards(
-        "api",
-        schema_editor,
-        None,
-        None,
-    )
-
-
-def _pg_sync_schema(apps, schema_editor):
-    from archivebox.misc.db import rebuild_models_from_migration_state
-
-    rebuild_models_from_migration_state(apps, schema_editor, "api", ["APIToken", "OutboundWebhook"])
-
-
-def _pg_drop_schema(apps, schema_editor):
-    from archivebox.misc.db import drop_models_on_postgres
-
-    drop_models_on_postgres(apps, schema_editor, "api", ["OutboundWebhook", "APIToken"])
-
-
-class Migration(migrations.Migration):
-    initial = True
-
-    dependencies = [
-        ("auth", "0012_alter_user_first_name_max_length"),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.SeparateDatabaseAndState(
-            database_operations=[
-                migrations.RunPython(_run_sqlite_only_sql, reverse_code=_run_sqlite_only_sql_reverse),
+                    """,
+                ),
             ],
             state_operations=[
                 migrations.CreateModel(
@@ -274,8 +236,4 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
-        # On non-sqlite backends the raw DDL above is skipped, so the real
-        # schema diverges from Django state. Rebuild both tables (including the
-        # unique constraint) from the post-migration state (no-op on sqlite).
-        migrations.RunPython(_pg_sync_schema, reverse_code=_pg_drop_schema),
     ]

archivebox/api/urls.py

@@ -23,4 +23,7 @@ urlpatterns = [
     path("v1/", RedirectView.as_view(url="/api/v1/docs")),
     path("v1/", v1_api_urls),
     path("v1", RedirectView.as_view(url="/api/v1/docs")),
+    # ... v2 can be added here ...
+    # path("v2/",              v2_api_urls),
+    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
 ]

archivebox/api/v1_api.py

@@ -2,8 +2,6 @@ __package__ = "archivebox.api"
 
 
 from io import StringIO
-from functools import lru_cache
-from typing import Any
 from traceback import format_exception
 from contextlib import redirect_stdout, redirect_stderr
 
@@ -12,7 +10,6 @@ from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, Permissio
 from django.contrib.auth.models import User
 
 from ninja import NinjaAPI, Swagger
-from ninja.openapi.schema import OpenAPISchema
 
 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
 
@@ -52,16 +49,6 @@ def register_urls(api: NinjaAPI) -> NinjaAPI:
 
 
 class NinjaAPIWithIOCapture(NinjaAPI):
-    def get_openapi_schema(self, *, path_prefix: str | None = None, path_params: dict[str, Any] | None = None) -> OpenAPISchema:
-        prefix = path_prefix if path_prefix is not None else self.get_root_path(path_params or {})
-        return self._schema_for_prefix(prefix)
-
-    @lru_cache(maxsize=8)
-    def _schema_for_prefix(self, path_prefix: str) -> OpenAPISchema:
-        # Routes are registered at startup. Server discovery can reuse their
-        # schema while preserving distinct URL prefixes for mounted instances.
-        return super().get_openapi_schema(path_prefix=path_prefix)
-
     def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
         stdout, stderr = StringIO(), StringIO()
 
@@ -91,6 +78,9 @@ class NinjaAPIWithIOCapture(NinjaAPI):
         response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
         response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
 
+        # import ipdb; ipdb.set_trace()
+        # print('RESPONDING NOW', response)
+
         return response
 
 
@@ -101,6 +91,8 @@ api = NinjaAPIWithIOCapture(
     auth=API_AUTH_METHODS,
     urls_namespace="api-1",
     docs=Swagger(settings={"persistAuthorization": True}),
+    # docs_decorator=login_required,
+    # renderer=ORJSONRenderer(),
 )
 api = register_urls(api)
 urls = api.urls
@@ -121,7 +113,24 @@ def generic_exception_handler(request, err):
             "message": f"{err.__class__.__name__}: {err}",
             "errors": [
                 "".join(format_exception(err)),
+                # or send simpler parent-only traceback:
+                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
             ],
         },
         status=status,
     )
+
+
+# import orjson
+# from ninja.renderers import BaseRenderer
+# class ORJSONRenderer(BaseRenderer):
+#     media_type = "application/json"
+#     def render(self, request, data, *, response_status):
+#         return {
+#             "success": True,
+#             "errors": [],
+#             "result": data,
+#             "stdout": ansi_to_html(stdout.getvalue().strip()),
+#             "stderr": ansi_to_html(stderr.getvalue().strip()),
+#         }
+#         return orjson.dumps(data)

archivebox/api/v1_auth.py

@@ -4,7 +4,7 @@ from django.http import HttpRequest
 
 from ninja import Router, Schema
 
-from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token, HeaderTokenAuth, BearerTokenAuth
+from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token
 
 
 router = Router(tags=["Authentication"], auth=None)
@@ -63,44 +63,3 @@ def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
         return {"success": True, "user_id": str(user.pk)}
 
     return {"success": False, "user_id": None}
-
-
-@router.post(
-    "/browser_session",
-    auth=[HeaderTokenAuth(), BearerTokenAuth()],
-    summary="Exchange an administrator API key for a normal browser session",
-)
-def browser_session(request: HttpRequest):
-    from django.conf import settings
-    from django.contrib.auth import login
-    from django.http import JsonResponse
-    from django.utils import timezone
-    from ninja.errors import HttpError
-    from archivebox.core.routes_util import get_admin_base_url
-
-    user = request.auth
-    if not user.is_active or not user.is_superuser:
-        raise HttpError(403, "An active administrator is required")
-    login(request, user, backend="django.contrib.auth.backends.ModelBackend")
-    # A browser session must not outlive the key used to establish it.
-    token = request._api_token
-    lifetime = settings.SESSION_COOKIE_AGE
-    if token.expires:
-        lifetime = min(lifetime, max(0, int((token.expires - timezone.now()).total_seconds())))
-    request.session.set_expiry(lifetime)
-    request.session.save()
-    response = JsonResponse(
-        {
-            "admin_url": get_admin_base_url(request=request).rstrip("/") + "/admin/",
-            "cookie": {
-                "name": settings.SESSION_COOKIE_NAME,
-                "value": request.session.session_key,
-                "expires": request.session.get_expiry_date().timestamp(),
-                "secure": settings.SESSION_COOKIE_SECURE,
-            },
-        },
-    )
-    # The native client installs a host-only HttpOnly cookie on the admin origin;
-    # API and admin hosts can differ, so a Set-Cookie on the API host is insufficient.
-    response["Cache-Control"] = "no-store"
-    return response

archivebox/api/v1_cli.py

@@ -15,6 +15,8 @@ from archivebox.misc.util import ansi_to_html
 from archivebox.core.models import SnapshotQuerySet
 
 
+# from .auth import API_AUTH_METHODS
+
 # router for API that exposes archivebox cli subcommands as REST endpoints
 router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 
@@ -52,7 +54,6 @@ class AddCommandSchema(Schema):
     snapshot_max_size: int = 0
     parser: str = "auto"
     plugins: str = ""
-    persona: str = "Default"
     only_new: bool | None = None
     update: bool = False
     overwrite: bool = False
@@ -120,30 +121,14 @@ def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
     from archivebox.cli.archivebox_add import add
-    from archivebox.misc.util import validate_url
 
     config_overrides: dict[str, object] = {}
     if args.only_new is not None:
         config_overrides["ONLY_NEW"] = bool(args.only_new)
     if args.update or args.overwrite:
         config_overrides["ONLY_NEW"] = False
-    submitted_urls: str | list[str] = args.urls
-    if len(args.urls) == 1:
-        try:
-            validate_url(args.urls[0])
-        except ValueError:
-            submitted_urls = args.urls[0]
-    else:
-        submitted_urls = []
-        for url in args.urls:
-            try:
-                submitted_urls.append(validate_url(url))
-            except ValueError:
-                continue
-        if not submitted_urls:
-            raise HttpError(400, "No valid URLs were submitted")
     crawl, snapshots = add(
-        urls=submitted_urls,
+        urls=args.urls,
         snapshot_ids=args.snapshot_ids,
         tag=args.tag,
         depth=args.depth,
@@ -153,7 +138,6 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         snapshot_max_size=args.snapshot_max_size,
         index_only=args.index_only,
         plugins=args.plugins,
-        persona=args.persona,
         parser=args.parser,
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,
@@ -165,7 +149,7 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         "crawl_id": str(crawl.id),
         "num_snapshots": len(snapshot_ids),
         "snapshot_ids": snapshot_ids,
-        "queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
+        "queued_urls": args.urls,
     }
     stdout = request.__dict__.get("stdout")
     stderr = request.__dict__.get("stderr")

archivebox/api/v1_core.py

@@ -4,14 +4,13 @@ import math
 import json
 import mimetypes
 import re
-from html import unescape
 from collections import defaultdict
 from pathlib import Path, PurePosixPath
 from uuid import UUID
 from typing import Union, Any, Annotated
 from datetime import datetime, time
 
-from django.db import IntegrityError
+from django.db import transaction
 from django.db.models import Model, Q
 from django.http import HttpRequest, HttpResponse
 from django.http.multipartparser import MultiPartParser, MultiPartParserError
@@ -32,7 +31,7 @@ from archivebox.core.models import Snapshot, ArchiveResult, Tag
 from archivebox.core.permissions import public_snapshots_queryset
 from archivebox.api.auth import authenticated_user_from_request
 from archivebox.config.common import get_config
-from archivebox.core.routes_util import build_snapshot_detail_url, build_web_url
+from archivebox.core.routes_util import build_web_url
 from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
 from archivebox.core.tag_util import (
     add_snapshot_counts,
@@ -50,7 +49,6 @@ from archivebox.core.tag_util import (
     rename_tag as rename_tag_record,
 )
 from archivebox.crawls.models import Crawl
-from archivebox.crawls.locks import crawl_lifecycle_lock
 from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref
 from archivebox.search.config import get_search_mode, get_search_mode_backend
 from archivebox.search.query import apply_snapshot_search
@@ -59,7 +57,7 @@ from archivebox.core.snapshot_status import filter_snapshots_by_status, normaliz
 
 router = Router(tags=["Core Models"])
 
-ARCHIVERESULT_UPLOAD_HOOK_NAME = Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME
+ARCHIVERESULT_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload"
 ARCHIVERESULT_UPLOAD_PLUGIN_RE = re.compile(r"^[A-Za-z0-9_.-]{1,32}$")
 
 
@@ -211,22 +209,11 @@ class ArchiveResultFilterSchema(FilterSchema):
     created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
 
 
-def _with_archiveresult_processes(queryset):
-    # cmd/pwd/version delegate to Process and Binary. Without this join a full
-    # capture's metadata response performs two extra indexed lookups per hook
-    # (roughly 100 queries), despite the initial Snapshot UUID lookup being fast.
-    # Logs and the hydrated hook environment are not response fields; loading
-    # those potentially large values would replace round trips with excess I/O.
-    return queryset.select_related("process__binary").defer("process__stdout", "process__stderr", "process__env")
-
-
 @router.get("/archiveresults", response=list[ArchiveResultSchema], url_name="get_archiveresult")
 @paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
     """List all ArchiveResult entries matching these filters."""
-    queryset = filters.filter(
-        _with_archiveresult_processes(ArchiveResult.objects.all()).prefetch_related("snapshot__crawl__created_by", "snapshot__tags"),
-    )
+    queryset = filters.filter(ArchiveResult.objects.all())
     if filters.search or filters.snapshot_tag:
         return queryset.distinct()
     return queryset
@@ -234,36 +221,31 @@ def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterS
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
     raw_ref = str(ref or "").strip()
+    query = Q(**{f"{field_name}__startswith": raw_ref})
+    if raw_ref:
+        query |= Q(**{f"{field_name}__icontains": raw_ref})
     try:
         parsed_uuid = UUID(raw_ref)
     except (TypeError, ValueError):
-        query = Q(**{f"{field_name}__startswith": raw_ref})
-        if raw_ref:
-            query |= Q(**{f"{field_name}__icontains": raw_ref})
         normalized_ref = raw_ref.replace("-", "")
         if normalized_ref and normalized_ref != raw_ref:
             query |= Q(**{f"{field_name}__startswith": normalized_ref})
             query |= Q(**{f"{field_name}__icontains": normalized_ref})
-        return query
-    # An exact UUID cannot need a substring fallback. OR-ing it with LIKE
-    # prevents an indexed lookup and made Cabbage metadata reads take seconds.
-    return Q(**{field_name: parsed_uuid})
+    else:
+        query |= Q(**{field_name: parsed_uuid})
+        query |= Q(**{f"{field_name}__startswith": parsed_uuid.hex})
+    return query
 
 
 @router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
 def get_archiveresult(request: HttpRequest, archiveresult_id: str):
     """Get a specific ArchiveResult by id."""
-    return (
-        _with_archiveresult_processes(ArchiveResult.objects.all())
-        .select_related("snapshot__crawl__created_by")
-        .prefetch_related("snapshot__tags")
-        .get(_uuid_ref_query("id", archiveresult_id))
-    )
+    return ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
 
 
 def _normalize_uploaded_archiveresult_plugin(plugin: str) -> str:
     normalized = str(plugin or "").strip().strip("/")
-    if normalized in (".", "..") or not ARCHIVERESULT_UPLOAD_PLUGIN_RE.fullmatch(normalized):
+    if not ARCHIVERESULT_UPLOAD_PLUGIN_RE.fullmatch(normalized):
         raise HttpError(400, "Invalid ArchiveResult plugin name")
     return normalized
 
@@ -358,20 +340,25 @@ def _parse_archiveresult_upload_int(value: str, field_name: str, *, default: int
 
 
 def _summarize_archiveresult_output_files(output_files: dict[str, dict[str, Any]]) -> tuple[int, str]:
-    from abx_dl.output_files import OutputManifest
-
-    manifest = OutputManifest.from_value(output_files)
-    return manifest.total_size, ",".join(manifest.mimetypes)
+    mime_sizes: dict[str, int] = defaultdict(int)
+    total_size = 0
+    for metadata in output_files.values():
+        if not isinstance(metadata, dict):
+            continue
+        try:
+            size = max(int(metadata.get("size") or 0), 0)
+        except (TypeError, ValueError):
+            size = 0
+        mime_type = str(metadata.get("mimetype") or "").strip()
+        total_size += size
+        if mime_type and size:
+            mime_sizes[mime_type] += size
+    output_mimetypes = ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
+    return total_size, output_mimetypes
 
 
 def _get_snapshot_by_ref(snapshot_id: str):
     queryset = Snapshot.objects.select_related("crawl__created_by")
-    try:
-        full_id = UUID(snapshot_id)
-    except (TypeError, ValueError):
-        pass
-    else:
-        return queryset.get(pk=full_id)
     try:
         return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
@@ -400,6 +387,13 @@ def _queue_archiveresult_snapshot_maintenance(snapshot: Snapshot) -> None:
     snapshot.safe_update(updates, refresh=False)
 
 
+def _merge_archiveresult_output_file_maps(results: list[ArchiveResult]) -> dict[str, dict[str, Any]]:
+    output_files: dict[str, dict[str, Any]] = {}
+    for result in results:
+        output_files.update(result.output_file_map())
+    return output_files
+
+
 def _write_archiveresult_files(
     request: HttpRequest,
     snapshot: Snapshot,
@@ -408,19 +402,18 @@ def _write_archiveresult_files(
     existing_output_files: dict[str, dict[str, Any]] | None = None,
     allow_empty: bool = False,
 ) -> dict[str, dict[str, Any]]:
-    plugin_name = _normalize_uploaded_archiveresult_plugin(plugin_name)
     files = _get_archiveresult_upload_files(request, allow_empty=allow_empty)
     output_paths = _get_archiveresult_upload_form_values(request, "output_paths", "output_path")
     mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
     chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
 
-    output_files = dict(existing_output_files or {})
-    if not files:
-        return output_files
-
     snapshot_dir = snapshot.output_dir
     plugin_dir = snapshot_dir / plugin_name
     storage = FileSystemStorage(location=str(plugin_dir))
+    output_files = dict(existing_output_files or {})
+
+    if not files:
+        return output_files
 
     if chunk_output_path:
         if len(files) != 1:
@@ -540,53 +533,67 @@ def create_archiveresult(
     normalized_status = ArchiveResult.normalize_status(status)
     parsed_output_json = _parse_archiveresult_output_json(output_json)
     hook = hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME
-    result_lookup = {
-        "snapshot": snapshot,
-        "plugin": plugin_name,
-        "hook_name": hook,
-    }
-    uploaded_output_files = _write_archiveresult_files(
+    matching_results = list(
+        ArchiveResult.objects.filter(
+            snapshot=snapshot,
+            plugin=plugin_name,
+            hook_name=hook,
+        ).order_by("created_at", "id"),
+    )
+    existing_result = matching_results[0] if matching_results else None
+    existing_output_files = _merge_archiveresult_output_file_maps(matching_results)
+    output_files = _write_archiveresult_files(
         request,
         snapshot,
         plugin_name,
+        existing_output_files=existing_output_files,
         allow_empty=True,
     )
-    result = ArchiveResult.objects.filter(**result_lookup).first()
-    for _attempt in range(3):
-        output_files = {
-            **(result.output_file_map() if result else {}),
-            **uploaded_output_files,
-        }
+    now = timezone.now()
+
+    with transaction.atomic():
+        matching_results = list(
+            ArchiveResult.objects.filter(
+                snapshot=snapshot,
+                plugin=plugin_name,
+                hook_name=hook,
+            ).order_by("created_at", "id"),
+        )
+        if matching_results:
+            existing_result = matching_results[0]
+            output_files = {
+                **_merge_archiveresult_output_file_maps(matching_results),
+                **output_files,
+            }
+            duplicate_ids = [result.id for result in matching_results[1:]]
+            if duplicate_ids:
+                ArchiveResult.objects.filter(id__in=duplicate_ids).delete()
+            result = existing_result
+        else:
+            existing_result = None
+            result = ArchiveResult(
+                snapshot=snapshot,
+                plugin=plugin_name,
+                hook_name=hook,
+            )
+
+        if (
+            existing_result
+            and normalized_status == ArchiveResult.StatusChoices.STARTED
+            and existing_result.status != ArchiveResult.StatusChoices.STARTED
+        ):
+            normalized_status = existing_result.status
         output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
         output_file_paths = list(output_files.keys())
-        result_status = normalized_status
-        if result and result_status == ArchiveResult.StatusChoices.STARTED and result.status != ArchiveResult.StatusChoices.STARTED:
-            result_status = result.status
-        now = timezone.now()
-        values = {
-            "status": result_status,
-            "output_str": output_str or (output_file_paths[0] if output_file_paths else ""),
-            "output_json": parsed_output_json,
-            "output_files": output_files,
-            "output_size": output_size,
-            "output_mimetypes": output_mimetypes,
-            "start_ts": result.start_ts or now if result else now,
-            "end_ts": now,
-        }
-        if result:
-            if result.safe_update(values):
-                break
-            continue
-        result, created = ArchiveResult.get_or_create_by_hook(
-            snapshot,
-            plugin_name,
-            hook,
-            defaults=values,
-        )
-        if created:
-            break
-    else:
-        raise HttpError(409, "ArchiveResult changed while upload metadata was being updated")
+        result.status = normalized_status
+        result.output_str = output_str or (output_file_paths[0] if output_file_paths else "")
+        result.output_json = parsed_output_json
+        result.output_files = output_files
+        result.output_size = output_size
+        result.output_mimetypes = output_mimetypes
+        result.start_ts = result.start_ts or now
+        result.end_ts = now
+        result.save()
 
     if result.status != ArchiveResult.StatusChoices.STARTED:
         _queue_archiveresult_snapshot_maintenance(snapshot)
@@ -600,46 +607,44 @@ def patch_archiveresult(
 ):
     """Append or replace files on an existing ArchiveResult."""
     result = ArchiveResult.objects.select_related("snapshot__crawl__created_by").get(_uuid_ref_query("id", archiveresult_id))
-    uploaded_output_files = _write_archiveresult_files(
+    output_files = _write_archiveresult_files(
         request,
         result.snapshot,
         result.plugin,
+        existing_output_files=result.output_file_map(),
     )
+    latest_result = ArchiveResult.objects.only("output_files", "status").get(pk=result.pk)
+    output_files = {
+        **latest_result.output_file_map(),
+        **output_files,
+    }
+    output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
+
+    update_fields = ["output_files", "output_size", "output_mimetypes", "end_ts", "modified_at"]
+    result.output_files = output_files
+    result.output_size = output_size
+    result.output_mimetypes = output_mimetypes
+    result.end_ts = timezone.now()
     output_str = _get_archiveresult_upload_form_value(request, "output_str")
     status = _get_archiveresult_upload_form_value(request, "status")
     output_json = _get_archiveresult_upload_form_value(request, "output_json")
-    parsed_output_json = _parse_archiveresult_output_json(output_json) if output_json else None
-
-    if not ArchiveResult.output_files_upload_complete(uploaded_output_files):
-        result.output_files = {**result.output_file_map(), **uploaded_output_files}
-        result.output_size, result.output_mimetypes = _summarize_archiveresult_output_files(result.output_files)
-        return result
-
-    for _attempt in range(3):
-        output_files = {**result.output_file_map(), **uploaded_output_files}
-        output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
-        values: dict[str, Any] = {
-            "output_files": output_files,
-            "output_size": output_size,
-            "output_mimetypes": output_mimetypes,
-            "end_ts": timezone.now(),
-        }
-        if output_str:
-            values["output_str"] = output_str
-        if status:
-            normalized_status = ArchiveResult.normalize_status(status)
-            if normalized_status == ArchiveResult.StatusChoices.STARTED and result.status != ArchiveResult.StatusChoices.STARTED:
-                normalized_status = result.status
-            values["status"] = normalized_status
-        elif result.status == ArchiveResult.StatusChoices.QUEUED:
-            values["status"] = ArchiveResult.StatusChoices.SUCCEEDED
-        if output_json:
-            values["output_json"] = parsed_output_json
-        if result.safe_update(values):
-            break
-    else:
-        raise HttpError(409, "ArchiveResult changed while upload metadata was being updated")
+    if output_str:
+        result.output_str = output_str
+        update_fields.append("output_str")
+    if status:
+        normalized_status = ArchiveResult.normalize_status(status)
+        if normalized_status == ArchiveResult.StatusChoices.STARTED and latest_result.status != ArchiveResult.StatusChoices.STARTED:
+            normalized_status = latest_result.status
+        result.status = normalized_status
+        update_fields.append("status")
+    elif latest_result.status == ArchiveResult.StatusChoices.QUEUED and ArchiveResult.output_files_upload_complete(output_files):
+        result.status = ArchiveResult.StatusChoices.SUCCEEDED
+        update_fields.append("status")
+    if output_json:
+        result.output_json = _parse_archiveresult_output_json(output_json)
+        update_fields.append("output_json")
 
+    result.save(update_fields=update_fields)
     if result.status != ArchiveResult.StatusChoices.STARTED:
         _queue_archiveresult_snapshot_maintenance(result.snapshot)
 
@@ -652,8 +657,6 @@ def patch_archiveresult(
 class SnapshotSchema(Schema):
     TYPE: str = "core.models.Snapshot"
     id: UUID
-    crawl_id: UUID
-    persona: str | None
     created_by_id: str
     created_by_username: str
     created_at: datetime
@@ -672,17 +675,6 @@ class SnapshotSchema(Schema):
     num_archiveresults: int
     archiveresults: list[MinimalArchiveResultSchema]
 
-    @staticmethod
-    def resolve_archive_path(obj):
-        return obj.archive_path_from_db
-
-    @staticmethod
-    def resolve_persona(obj):
-        # A deduplicated submission can reuse a capture from a different
-        # persona. Report its actual DB-backed owner, not the new request's choice.
-        persona = obj.crawl.resolve_persona()
-        return persona.name if persona else None
-
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by.pk)
@@ -705,14 +697,12 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_num_archiveresults(obj, context):
-        # This reverse FK contains each result once. DISTINCT would materialize
-        # wide JSON output columns just to count rows through snapshot_id's index.
-        return obj.archiveresult_set.count()
+        return obj.archiveresult_set.all().distinct().count()
 
     @staticmethod
     def resolve_archiveresults(obj, context):
         if bool(context["request"].__dict__.get("with_archiveresults", False)):
-            return _with_archiveresult_processes(obj.archiveresult_set.all())
+            return obj.archiveresult_set.all().distinct()
         return ArchiveResult.objects.none()
 
 
@@ -737,7 +727,6 @@ class SnapshotDeleteResponseSchema(Schema):
     snapshot_id: str
     crawl_id: str
     deleted_count: int
-    queued_count: int
 
 
 def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
@@ -838,12 +827,12 @@ def _snapshots_rss_response(
     )
 
     for snapshot in snapshots:
-        archived_url = build_snapshot_detail_url(snapshot.archive_path_from_db, request=request)
+        archived_url = build_web_url(f"/{snapshot.archive_path_from_db}", request=request)
         tags = [tag.name for tag in snapshot.tags.all()]
         crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
         description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
         feed.add_item(
-            title=unescape(snapshot.title or snapshot.url),
+            title=snapshot.title or snapshot.url,
             link=archived_url or web_base_url,
             description=description,
             unique_id=str(snapshot.id),
@@ -972,51 +961,37 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    # Browser uploads must not wait behind the runner's crawl-wide lifecycle
-    # lock. The unique insert recovery and CAS update below are the request-side
-    # coordination boundary for this idempotent metadata sync.
-    snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-    if snapshot is None and bool(crawl.get_current_config().get("ONLY_NEW", True)):
-        # Uploading browser metadata is part of the same submission as /add.
-        # It must honor the runner's deduplication policy, even if it arrives
-        # before the runner has processed that crawl. Report the actual owner
-        # so clients never mistake a reused capture for one they just created.
-        snapshot = Snapshot.objects.filter(url=data.url).exclude(status=Snapshot.DELETING_STATE).order_by("-created_at").first()
-        if snapshot is not None:
-            setattr(request, "with_archiveresults", False)
-            return snapshot
-    if snapshot is None:
-        try:
-            snapshot = Snapshot.objects.create(
-                url=data.url,
-                crawl=crawl,
-                depth=data.depth,
-                title=data.title,
-                timestamp=str(timezone.now().timestamp()),
-                status=status or Snapshot.StatusChoices.QUEUED,
-                retry_at=timezone.now(),
-            )
-        except IntegrityError:
-            snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-            if snapshot is None:
-                raise
+    snapshot_defaults = {
+        "depth": data.depth,
+        "title": data.title,
+        "timestamp": str(timezone.now().timestamp()),
+        "status": status or Snapshot.StatusChoices.QUEUED,
+        "retry_at": timezone.now(),
+    }
+    snapshot, _ = Snapshot.objects.get_or_create(
+        url=data.url,
+        crawl=crawl,
+        defaults=snapshot_defaults,
+    )
 
-    for _attempt in range(3):
-        updates: dict[str, Any] = {}
-        if data.title is not None and snapshot.title != data.title:
-            updates["title"] = data.title
-        if status is not None and snapshot.status != status:
-            updates["status"] = status
-        if not updates or snapshot.safe_update(updates, extra_filter={"modified_at": snapshot.modified_at}):
-            break
-    else:
-        raise HttpError(409, "Snapshot changed while metadata was being updated")
+    update_fields: list[str] = []
+    if data.title is not None and snapshot.title != data.title:
+        snapshot.title = data.title
+        update_fields.append("title")
+    if status is not None and snapshot.status != status:
+        snapshot.status = status
+        update_fields.append("status")
+    if update_fields:
+        update_fields.append("modified_at")
+        snapshot.save(update_fields=update_fields)
 
     if tags:
-        snapshot.save_tags(
-            tags,
-            created_by=request.user if isinstance(request.user, User) else None,
-        )
+        snapshot.save_tags(tags)
+
+    try:
+        snapshot.ensure_crawl_symlink()
+    except Exception:
+        pass
 
     setattr(request, "with_archiveresults", False)
     return snapshot
@@ -1026,7 +1001,6 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
 def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
     """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
     snapshot = _get_snapshot_by_ref(snapshot_id)
-    crawl_id = str(snapshot.crawl_id)
 
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
@@ -1048,54 +1022,41 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
             return snapshot
         raise HttpError(400, f"Invalid action: {action}")
 
-    with crawl_lifecycle_lock(crawl_id):
-        snapshot = _get_snapshot_by_ref(snapshot_id)
-        if "status" in payload:
-            try:
-                snapshot.status = normalize_snapshot_status(payload["status"])
-            except ValueError as err:
-                raise HttpError(400, str(err)) from err
-            if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
-                snapshot.retry_at = None
-            update_fields.append("status")
+    if "status" in payload:
+        try:
+            snapshot.status = normalize_snapshot_status(payload["status"])
+        except ValueError as err:
+            raise HttpError(400, str(err)) from err
+        if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
+            snapshot.retry_at = None
+        update_fields.append("status")
 
-        if "retry_at" in payload:
-            snapshot.retry_at = payload["retry_at"]
-            update_fields.append("retry_at")
+    if "retry_at" in payload:
+        snapshot.retry_at = payload["retry_at"]
+        update_fields.append("retry_at")
 
-        if tags is not None:
-            snapshot.save_tags(
-                normalize_tag_list(tags),
-                created_by=request.user if isinstance(request.user, User) else None,
-            )
+    if tags is not None:
+        snapshot.save_tags(normalize_tag_list(tags))
 
-        if payload.get("status") == Snapshot.StatusChoices.SEALED:
-            snapshot.cancel()
-        else:
-            snapshot.save(update_fields=update_fields)
+    if payload.get("status") == Snapshot.StatusChoices.SEALED:
+        snapshot.cancel()
+    else:
+        snapshot.save(update_fields=update_fields)
     setattr(request, "with_archiveresults", False)
     return snapshot
 
 
 @router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
 def delete_snapshot(request: HttpRequest, snapshot_id: str):
-    """Queue deletion; the normal server runner stops writers and removes files.
-
-    The snapshot remains readable with status=deleting until cleanup succeeds.
-    queued_count reports acceptance; deleted_count is zero in this response.
-    """
     snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
     snapshot_id_str = str(snapshot.id)
-    crawl_id_str = str(snapshot.crawl_id)
-    # Use the same durable queue as the admin delete action. Stopping writers
-    # and removing archive files belongs to the runner, never the HTTP request.
-    queued_count = Snapshot.objects.filter(pk=snapshot.pk).request_delete()
+    crawl_id_str = str(snapshot.crawl.pk)
+    deleted_count, _ = snapshot.delete()
     return {
         "success": True,
         "snapshot_id": snapshot_id_str,
         "crawl_id": crawl_id_str,
-        "deleted_count": 0,
-        "queued_count": queued_count,
+        "deleted_count": deleted_count,
     }
 
 
@@ -1455,7 +1416,8 @@ def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
     else:
         raise HttpError(400, "Either tag_name or tag_id is required")
 
-    snapshot.add_tag_ids([tag.pk])
+    # Add the tag to the snapshot
+    snapshot.tags.add(tag.pk)
 
     return {
         "success": True,
@@ -1483,7 +1445,8 @@ def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSche
     else:
         raise HttpError(400, "Either tag_name or tag_id is required")
 
-    snapshot.remove_tag_ids([tag.pk])
+    # Remove the tag from the snapshot
+    snapshot.tags.remove(tag.pk)
 
     return {
         "success": True,

archivebox/api/v1_crawls.py

@@ -22,7 +22,7 @@ from archivebox.core.permissions import (
 )
 from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
-from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url
+from archivebox.misc.util import filter_queryset_by_uuid_substring
 
 from .auth import API_AUTH_METHODS, authenticated_user_from_request
 
@@ -48,6 +48,8 @@ class CrawlSchema(Schema):
     tags_str: str
     config: dict
 
+    # snapshots: List[SnapshotSchema]
+
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
@@ -112,10 +114,7 @@ def get_crawls(request: HttpRequest):
 
 @router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
 def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
-    try:
-        urls = [validate_url(url) for url in data.urls if url and url.strip()]
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
+    urls = [url.strip() for url in data.urls if url and url.strip()]
     if not urls:
         raise HttpError(400, "At least one URL is required")
     if data.max_depth not in (0, 1, 2, 3, 4):
@@ -123,7 +122,6 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
 
     tags = normalize_tag_list(data.tags, data.tags_str)
     config = dict(data.config or {})
-    config.setdefault("PARSER", "url_list")
     config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
     crawl = Crawl.objects.create(
         urls="\n".join(urls),
@@ -182,14 +180,6 @@ def crawl_file(request: HttpRequest, crawl_id: str, path: str):
     if not file_path.is_file() or crawl_root not in file_path.parents:
         raise HttpError(404, "Crawl file not found")
 
-    if path == "chrome_screencast/latest.jpg" and request.GET.get("after"):
-        try:
-            started_at = float(request.GET["after"])
-        except ValueError:
-            raise HttpError(400, "Invalid screencast start time")
-        if file_path.stat().st_mtime < started_at:
-            raise HttpError(404, "No frame for this capture yet")
-
     response = FileResponse(file_path.open("rb"))
     response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
     response["Pragma"] = "no-cache"
@@ -260,16 +250,8 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
 def delete_crawl(request: HttpRequest, crawl_id: str):
     crawl = get_crawl_by_ref(crawl_id)
     crawl_id_str = str(crawl.id)
-    crawl.cancel()
-
-    from archivebox.crawls.locks import crawl_lifecycle_lock
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        crawl = get_crawl_by_ref(crawl_id_str)
-        snapshot_count = crawl.snapshot_set.count()
-        deleted_count, _ = crawl.delete()
+    snapshot_count = crawl.snapshot_set.count()
+    deleted_count, _ = crawl.delete()
     return {
         "success": True,
         "crawl_id": crawl_id_str,

archivebox/api/v1_personas.py

@@ -2,14 +2,12 @@ __package__ = "archivebox.api"
 
 import json
 from datetime import datetime
-from typing import Any, Literal
+from typing import Any
 from uuid import UUID
 
-from django.core.exceptions import ValidationError
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
-from ninja.errors import HttpError
 from ninja.pagination import paginate
 from pydantic import Field
 
@@ -25,7 +23,6 @@ class PersonaBrowserSettingsSchema(Schema):
     user_agent: str = ""
     viewport_size: str = ""
     viewport_device_scale_factor: float | None = None
-    color_scheme: Literal["", "light", "dark"] = ""
     language: str = ""
     timezone: str = ""
     geolocation: dict[str, Any] | None = None
@@ -97,8 +94,6 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
         )
     if settings.viewport_device_scale_factor is not None:
         config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
-    if "color_scheme" in settings.model_fields_set:
-        config["BROWSER_COLOR_SCHEME"] = settings.color_scheme
     if settings.language:
         config["BROWSER_LANGUAGE"] = settings.language
     if settings.timezone:
@@ -110,9 +105,10 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
 
 
 def find_persona(extension_persona_id: str, name: str) -> Persona | None:
-    named = Persona.find_named(name)
     return (
-        Persona.objects.filter(Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(pk=named.pk if named else None))
+        Persona.objects.filter(
+            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
+        )
         .order_by("created_at")
         .first()
     )
@@ -139,11 +135,7 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     if not is_valid:
         raise ValueError(error_message)
 
-    try:
-        persona = find_persona(payload.extension_persona_id, name)
-        (persona or Persona(name=name)).validate_name(persona.name if persona else name)
-    except ValidationError as err:
-        raise HttpError(409, "; ".join(err.messages)) from err
+    persona = find_persona(payload.extension_persona_id, name)
     created = persona is None
     if persona is None:
         persona = Persona(name=name)
@@ -158,12 +150,12 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     persona.ensure_dirs()
 
     cookies_written = False
-    if "cookies_txt" in payload.model_fields_set:
+    if payload.cookies_txt.strip():
         (persona.path / "cookies.txt").write_text(payload.cookies_txt)
         cookies_written = True
 
     auth_written = False
-    if "auth_json" in payload.model_fields_set:
+    if payload.auth_json:
         (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
         auth_written = True
 

archivebox/api/webhooks.py

@@ -5,40 +5,12 @@ from collections.abc import Callable
 import logging
 
 from django.db import transaction
-from django.db.models.signals import pre_delete
 from signal_webhooks.handlers import sync_task_handler
 
 
 logger = logging.getLogger(__name__)
 
 
-def load_deferred_fields_for_delete_webhook(sender, instance, using, **kwargs) -> None:
-    """Materialize deferred fields while DELETE webhook rows still exist.
-
-    Admin lists defer large fields, but signal_webhooks serializes all concrete
-    fields in post_delete. Loading one then would query an already-deleted row
-    and drop the event. Hydrate only deferred fields on webhook-enabled models
-    here, preserving the normal list-query optimization.
-    """
-    from signal_webhooks.handlers import find_hook_handler
-    from signal_webhooks.utils import reference_for_model
-
-    ref = reference_for_model(type(instance))
-    deferred_fields = instance.get_deferred_fields()
-    if not deferred_fields or find_hook_handler(ref, "DELETE") is None:
-        return
-
-    instance.refresh_from_db(using=using, fields=deferred_fields)
-
-
-def register_delete_webhook_field_loader() -> None:
-    pre_delete.connect(
-        load_deferred_fields_for_delete_webhook,
-        dispatch_uid="archivebox.load_deferred_fields_for_delete_webhook",
-        weak=False,
-    )
-
-
 def warning_error_handler(hook: Any, error: Exception | None) -> None:
     if error is not None:
         logger.warning("Outbound webhook %r failed: %s", hook.name, error)

archivebox/base_models/admin.py

@@ -5,11 +5,11 @@ __package__ = "archivebox.base_models"
 import json
 import uuid
 from collections.abc import Mapping
-from typing import ClassVar, NotRequired, TypedDict, cast
+from typing import NotRequired, TypedDict, cast
 
 from django import forms
 from django.contrib import admin
-from django.db import DatabaseError, models
+from django.db import models
 from django.forms.renderers import BaseRenderer
 from django.http import HttpRequest, QueryDict
 from django.urls import path, register_converter
@@ -68,10 +68,10 @@ class KeyValueWidget(forms.Widget):
     template_name = ""  # We render manually
 
     class Media:
-        css: ClassVar[dict[str, list[str]]] = {
+        css = {
             "all": [],
         }
-        js: ClassVar[list[str]] = []
+        js = []
 
     def _get_config_options(self) -> dict[str, ConfigOption]:
         """Get available config options from plugins."""
@@ -94,7 +94,7 @@ class KeyValueWidget(forms.Widget):
                             option[schema_key] = schema[schema_key]
                 options[key] = option
             return options
-        except (ImportError, KeyError, TypeError, ValueError):
+        except Exception:
             return {}
 
     def _parse_value(self, value: object) -> dict[str, object]:
@@ -644,64 +644,41 @@ class KeyValueWidget(forms.Widget):
 
                     window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
 
-                    function configRowForKey_{widget_id}(key) {{
+                    function focusConfigKeyFromHash_{widget_id}() {{
+                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
+                        // to (or creates) the matching row in this editor. Used by
+                        // the in-banner "pin via admin" link and the
+                        // ``→ Edit <KEY> in Machine.config`` shortcut on the live
+                        // config detail page.
+                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
+                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
+                            return;
+                        }}
                         var container = document.getElementById('{widget_id}_rows');
                         if (!container) {{
-                            return null;
+                            return;
                         }}
                         var match = null;
                         container.querySelectorAll('.key-value-row').forEach(function(row) {{
                             if (match) {{ return; }}
                             var keyInput = row.querySelector('.kv-key');
-                            if (keyInput && keyInput.value.trim() === key) {{
+                            if (keyInput && keyInput.value.trim() === hash) {{
                                 match = row;
                             }}
                         }});
                         if (!match) {{
+                            // No existing row for this key — prepopulate one with the
+                            // key filled in but value left blank so the operator just
+                            // types/pastes the value and hits save.
                             window.addKeyValueRow_{widget_id}();
                             var rows = container.querySelectorAll('.key-value-row');
                             match = rows[rows.length - 1];
                             var keyInput = match.querySelector('.kv-key');
                             if (keyInput) {{
-                                keyInput.value = key;
+                                keyInput.value = hash;
                                 keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
                             }}
                         }}
-                        return match;
-                    }}
-
-                    function prefillConfigFromQuery_{widget_id}() {{
-                        var params = new URLSearchParams(window.location.search);
-                        var consumedConfigKeys = [];
-                        params.forEach(function(value, key) {{
-                            if (!/^[A-Z][A-Z0-9_]*$/.test(key) || !configMeta_{widget_id}[key]) {{
-                                return;
-                            }}
-                            consumedConfigKeys.push(key);
-                            var match = configRowForKey_{widget_id}(key);
-                            var valueInput = match && match.querySelector('.kv-value');
-                            if (valueInput) {{
-                                valueInput.value = value;
-                                valueInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
-                            }}
-                        }});
-                        updateHiddenField_{widget_id}();
-                        if (consumedConfigKeys.length) {{
-                            var cleanUrl = new URL(window.location.href);
-                            consumedConfigKeys.forEach(function(key) {{ cleanUrl.searchParams.delete(key); }});
-                            window.history.replaceState(null, '', cleanUrl.pathname + cleanUrl.search + cleanUrl.hash);
-                        }}
-                    }}
-
-                    function focusConfigKeyFromHash_{widget_id}() {{
-                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
-                        // to (or creates) the matching row in this editor. Used by
-                        // the setup wizard and live-config detail pages.
-                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
-                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
-                            return;
-                        }}
-                        var match = configRowForKey_{widget_id}(hash);
                         if (!match) {{
                             return;
                         }}
@@ -723,14 +700,12 @@ class KeyValueWidget(forms.Widget):
                     // Initialize on load
                     document.addEventListener('DOMContentLoaded', function() {{
                         initializeRows_{widget_id}();
-                        prefillConfigFromQuery_{widget_id}();
                         updateHiddenField_{widget_id}();
                         focusConfigKeyFromHash_{widget_id}();
                     }});
                     // Also run immediately in case DOM is already ready
                     if (document.readyState !== 'loading') {{
                         initializeRows_{widget_id}();
-                        prefillConfigFromQuery_{widget_id}();
                         updateHiddenField_{widget_id}();
                         focusConfigKeyFromHash_{widget_id}();
                     }}
@@ -851,7 +826,7 @@ class ConfigEditorMixin(admin.ModelAdmin):
         if change and obj.pk and obj.config is not None:
             try:
                 stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
-            except (AttributeError, DatabaseError, TypeError, ValueError):
+            except Exception:
                 stored = {}
             if isinstance(stored, dict):
                 new_config = dict(obj.config or {})
@@ -870,20 +845,6 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
     list_display = ("id", "created_at", "created_by")
     readonly_fields = ("id", "created_at", "modified_at")
     show_search_mode_selector = False
-    change_form_template = "admin/archivebox_change_form.html"
-
-    def get_admin_toolbar_actions(self, request, obj):
-        """Return extra action button dicts for the shared change-form toolbar.
-
-        See ``templates/admin/includes/archivebox_toolbar.html`` for the
-        accepted keys. Default: no extras (toolbar renders Save/History/Delete
-        plus any ``django-object-actions`` change_actions).
-        """
-        return []
-
-    def render_change_form(self, request, context, add=False, change=False, form_url="", obj=None):
-        context.setdefault("archivebox_admin_actions", self.get_admin_toolbar_actions(request, obj))
-        return super().render_change_form(request, context, add=add, change=change, form_url=form_url, obj=obj)
 
     def get_default_search_mode(self) -> str:
         # The shared changelist template always asks every admin for a default
@@ -916,9 +877,9 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
         """
         info = self.opts.app_label, self.opts.model_name
         object_routes = [
-            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="{}_{}_history".format(*info)),
-            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="{}_{}_delete".format(*info)),
-            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="{}_{}_change".format(*info)),
+            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="%s_%s_history" % info),
+            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="%s_%s_delete" % info),
+            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="%s_%s_change" % info),
         ]
         # Append after super().get_urls() so our patterns are the
         # *last-registered* ones with the canonical admin URL names — Django's

archivebox/base_models/apps.py

@@ -0,0 +1,7 @@
+# from django.apps import AppConfig
+
+
+# class BaseModelsConfig(AppConfig):
+#     default_auto_field = 'django.db.models.BigAutoField'
+
+#     name = 'base_models'

archivebox/base_models/models.py

@@ -9,7 +9,6 @@ from typing import Any
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 
-from django.core.exceptions import FieldDoesNotExist
 from django.db import models
 from django.db.models import F
 from django.db import transaction
@@ -24,110 +23,6 @@ from django_stubs_ext.db.models import TypedModelMeta
 from archivebox.config import CONSTANTS
 
 
-class PagedQuerySet(models.QuerySet):
-    def paged_iterator(self, chunk_size: int = 500):
-        """
-        Iterate rows using bounded keyset pages instead of one streaming cursor.
-
-        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
-        cursor open until the full queryset is exhausted. That is fine for
-        read-only exports, but update/migration code does filesystem work and
-        writes while iterating; a long-lived read cursor there can stretch lock
-        waits across thousands of rows. This respects the queryset's existing
-        filters, order_by(), select_related(), and prefetch_related() state; if
-        no ordering is defined, it falls back to primary-key order.
-        """
-        pk_field = self.model._meta.pk.name
-        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
-
-        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        ordering = []
-        for term in raw_ordering:
-            descending = term.startswith("-")
-            field_name = term[1:] if descending else term
-            if field_name == "pk":
-                field_name = pk_field
-            ordering.append(f"-{field_name}" if descending else field_name)
-
-        ordered_field_names = [term.removeprefix("-") for term in ordering]
-        try:
-            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
-                offset = 0
-                while True:
-                    batch = list(self[offset : offset + chunk_size])
-                    if not batch:
-                        break
-                    yield from batch
-                    offset += chunk_size
-                return
-        except (AttributeError, FieldDoesNotExist):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
-        if not any(field_name in unique_field_names for field_name in ordered_field_names):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        last_values = None
-        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
-        while True:
-            batch_qs = self.order_by(*ordering)
-            if last_values is not None:
-                page_filter = models.Q()
-                for idx, term in enumerate(ordering):
-                    descending = term.startswith("-")
-                    field_name = term[1:] if descending else term
-                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
-                    comparison = "lt" if descending else "gt"
-                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
-                # Anchor the OR tie-breaks with an explicit leading range so
-                # SQLite/PG can seek the existing order index instead of
-                # rescanning its prefix for every keyset page.
-                first_comparison = "lte" if ordering[0].startswith("-") else "gte"
-                batch_qs = batch_qs.filter(
-                    page_filter,
-                    **{f"{ordered_field_names[0]}__{first_comparison}": last_values[0]},
-                )
-
-            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
-            if not batch_rows:
-                break
-
-            pk_idx = value_field_names.index(pk_field)
-            row_ids = [row[pk_idx] for row in batch_rows]
-            rows_by_id = {obj.pk: obj for obj in self.filter(pk__in=row_ids).order_by()}
-
-            for row in batch_rows:
-                row_id = row[pk_idx]
-                obj = rows_by_id.get(row_id)
-                if obj is not None:
-                    yield obj
-
-            last_values = batch_rows[-1][: len(ordered_field_names)]
-
-
 def normalize_config_json_values(config: Any) -> Any:
     if not isinstance(config, dict):
         return config
@@ -296,21 +191,9 @@ class ModelWithDeleteAfter(models.Model):
         return cls.objects.none()
 
     @classmethod
-    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True, backfill_cursors: dict | None = None) -> int:
+    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
         if backfill_missing:
-            # Bound the scanned rows BEFORE evaluating JSON policy/parent
-            # joins. LIMIT on the policy query only bounded its output: finding
-            # no matches could scan the entire history with a SQLite read lock.
-            # The runner keeps a keyset cursor between idle passes, so disabled
-            # policies cannot starve older rows. Restarting safely repeats work.
-            page = cls.objects.order_by("-pk")
-            before = backfill_cursors.get(cls) if backfill_cursors is not None else None
-            if before is not None:
-                page = page.filter(pk__lt=before)
-            page_ids = list(page.values_list("pk", flat=True)[:batch_size])
-            if backfill_cursors is not None:
-                backfill_cursors[cls] = page_ids[-1] if len(page_ids) == batch_size else None
-            missing_delete_at = list(cls.missing_delete_at_candidates().filter(pk__in=page_ids).order_by())
+            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
             for obj in missing_delete_at:
                 if obj.set_delete_at_from_config():
                     cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
@@ -348,9 +231,12 @@ class ModelWithOutputDir(ModelWithUUID):
 
     _delete_signal_registered = False
 
-    # Saving metadata must not resolve or create output paths. The runner and
-    # explicit artifact writers create their own directories when writing files;
-    # on_commit would still make HTTP requests wait on remote archive storage.
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        output_dir = Path(self.output_dir)
+        # Avoid holding SQLite write transactions open across slow filesystem work.
+        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
+        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
 
     @property
     def output_dir_parent(self) -> str:
@@ -396,31 +282,18 @@ class ModelWithOutputDir(ModelWithUUID):
             if path.is_symlink() or path.is_file():
                 path.unlink(missing_ok=True)
             elif path.is_dir():
-                shutil.rmtree(path)
-            # lstat also catches dangling symlinks and propagates mount and
-            # permission errors rather than mistaking them for absence.
-            try:
-                path.lstat()
-            except FileNotFoundError:
-                continue
-            raise OSError(f"Output path still exists after deletion: {path}")
-
-    def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
-        """Capture output paths before DB deletion and remove them after commit."""
-        paths = self.validate_output_paths_for_delete(self.output_paths_for_delete())
-        transaction.on_commit(lambda: self.delete_output_paths(paths), using=using)
+                shutil.rmtree(path, ignore_errors=True)
 
     @classmethod
     def register_delete_signal(cls) -> None:
         if cls._delete_signal_registered:
             return
 
-        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
-            if getattr(origin, "_output_files_deleted", False):
-                return
+        def schedule_output_dir_cleanup(sender, instance, **kwargs):
             if not isinstance(instance, ModelWithOutputDir):
                 return
-            instance.schedule_delete_cleanup(using=using)
+            paths = instance.validate_output_paths_for_delete(instance.output_paths_for_delete())
+            transaction.on_commit(lambda paths=paths: instance.delete_output_paths(paths))
 
         pre_delete.connect(
             schedule_output_dir_cleanup,

archivebox/cli/__init__.py

@@ -2,7 +2,6 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox"
 import os
 import sys
-from contextlib import redirect_stdout
 from importlib import import_module
 
 import rich_click as click
@@ -19,11 +18,10 @@ if "--debug" in sys.argv:
 
 # Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
 # `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
-# the subcommand executes. `--quick-init` is the old server spelling. Strip either from
-# argv here so each subcommand's own click parser never sees it. Ignored for `help` and
-# `init` themselves.
-if "--init" in sys.argv or "--quick-init" in sys.argv:
-    sys.argv = [arg for arg in sys.argv if arg not in ("--init", "--quick-init")]
+# the subcommand executes. Strip it from argv here so each subcommand's own click parser
+# never sees it. Ignored for `help` and `init` themselves.
+if "--init" in sys.argv:
+    sys.argv = [arg for arg in sys.argv if arg != "--init"]
     os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
 
 
@@ -34,7 +32,6 @@ class ArchiveBoxGroup(click.Group):
         "help": "archivebox.cli.archivebox_help.main",
         "version": "archivebox.cli.archivebox_version.main",
         "mcp": "archivebox.cli.archivebox_mcp.main",
-        "oneshot": "archivebox.cli.archivebox_oneshot.main",
     }
     setup_commands = {
         "init": "archivebox.cli.archivebox_init.main",
@@ -129,12 +126,16 @@ class ArchiveBoxGroup(click.Group):
             import_path = cmd_name_or_path
         modname, funcname = import_path.rsplit(".", 1)
 
+        # print(f'LAZY LOADING {import_path}')
         mod = import_module(modname)
         func = vars(mod)[funcname]
 
         if func.__doc__ is None:
             raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
 
+        # if not isinstance(cmd, click.BaseCommand):
+        # raise ValueError(f'lazy loading of {import_path} failed - not a click command')
+
         return func
 
 
@@ -155,11 +156,15 @@ def cli(ctx, help=False):
     # then we need to set up the django environment and check that we're in a valid data folder
     wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
     if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
+        # print('SETUP DJANGO AND CHECK DATA FOLDER')
         try:
-            # RUNSERVER_WORKER sets the web-worker markers on its own child.
-            # Setting them on the server parent leaks them through supervisord
-            # into runners waiting for takeover, which then register themselves
-            # as web workers and trigger spurious runner_watch reloads.
+            if subcommand == "server":
+                run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes")
+                if run_in_debug:
+                    os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
+                    if "--reload" in sys.argv:
+                        os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
+
             from archivebox.config.django import setup_django
             from archivebox.misc.checks import check_data_folder, check_migrations
 
@@ -175,12 +180,8 @@ def cli(ctx, help=False):
                 archivebox_init(quick=True)
                 os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
             check_data_folder()
-            if subcommand != "update" and not check_migrations(auto_apply=True):
-                from archivebox.machine.models import Machine
-
-                # Host registration and config reconciliation belong to CLI
-                # startup; get_config() is also used by read-only HTTP views.
-                Machine.current()
+            if subcommand != "update":
+                check_migrations(auto_apply=True)
         except Exception as e:
             STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
             if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
@@ -188,10 +189,6 @@ def cli(ctx, help=False):
 
 
 def main(args=None, prog_name=None):
-    from archivebox.config.constants import CONSTANTS
-
-    os.environ.setdefault("ABX_PLUGINS_DIR", str(CONSTANTS.USER_PLUGINS_DIR))
-
     # show `docker run archivebox xyz` in help messages if running in docker
     IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
     IS_TTY = sys.stdin.isatty()
@@ -220,12 +217,6 @@ def main(args=None, prog_name=None):
         raise SystemExit(130) from None
     finally:
         sys.unraisablehook = previous_unraisablehook
-        supervisor_util = sys.modules.get("archivebox.workers.supervisord_util")
-        if supervisor_util is not None:
-            # Short commands can start a foreground-owned search daemon too.
-            # Finish its cleanup before another CLI can borrow a dying stack.
-            with redirect_stdout(sys.stderr):
-                supervisor_util.stop_own_supervisord_process()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_add.py

@@ -6,7 +6,6 @@ __command__ = "archivebox add"
 
 import sys
 import os
-import signal
 from pathlib import Path
 
 from typing import Any, TYPE_CHECKING
@@ -71,7 +70,17 @@ def add(
     created_by_id: int | None = None,
     config: dict[str, Any] | None = None,
 ) -> tuple[Crawl, QuerySet[Snapshot]]:
-    """Add a URL list or imported URL document to a new Crawl."""
+    """Add a new URL or list of URLs to your archive.
+
+    The flow is:
+    1. Save URLs to sources file
+    2. Create Crawl with URLs and max_depth
+    3. Crawl runner creates Snapshots from Crawl URLs (depth=0)
+    4. Crawl runner runs parser extractors on root snapshots
+    5. Parser extractors output to urls.jsonl
+    6. URLs are added to Crawl.urls and child Snapshots are created
+    7. Repeat until max_depth is reached
+    """
 
     from rich import print
 
@@ -105,6 +114,7 @@ def add(
 
     # import models once django is set up
     from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.personas.models import Persona
     from archivebox.misc.logging_util import printable_filesize
@@ -115,25 +125,9 @@ def add(
     created_by_id = created_by_id or get_or_create_system_user_pk()
     started_at = timezone.now()
 
-    if isinstance(urls, str):
-        source_text = urls
-    else:
-        from archivebox.misc.util import validate_url
-
-        source_text = "\n".join(validate_url(str(url)) for url in urls)
+    source_text = urls if isinstance(urls, str) else "\n".join(str(url) for url in urls)
 
     # 2. Create a new Crawl with inline URLs
-    # Foreground add must claim runner ownership before publishing runnable
-    # work. Otherwise an existing server runner can lease the new Crawl in the
-    # gap between Crawl.objects.create() and current_command(), then be killed
-    # by the newer add process halfway through its first Snapshot hook.
-    command = None
-    if not bg and not index_only:
-        from archivebox.machine.models import Process
-        from archivebox.core.takeover_util import current_command
-
-        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR)
-
     cli_args = [*sys.argv]
     if cli_args[0].lower().endswith("archivebox"):
         cli_args[0] = "archivebox"
@@ -144,6 +138,7 @@ def add(
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
     persona_obj = Persona.get_or_create_named(persona_name)
+    persona_obj.ensure_dirs()
     effective_persona_config = get_config(persona=persona_obj)
 
     crawl_config = {
@@ -173,32 +168,41 @@ def add(
     # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
     # callers audit or resume imports without losing RSS/Netscape/JSON metadata
     # that is not representable as one plain URL per line.
-    try:
-        crawl = Crawl.objects.create(
-            urls=source_text,
-            max_depth=depth,
-            tags_str=tag,
-            persona_id=persona_obj.id,
-            label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
-            created_by_id=created_by_id,
-            status=Crawl.StatusChoices.QUEUED,
-            retry_at=None if index_only else timezone.now(),
-            config=crawl_config,
-        )
-        first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
-    except BaseException:
-        if command is not None:
-            command.mark_exited(exit_code=1)
-        raise
-    if command is not None:
-        command.url = first_url
-        command.save(update_fields=["url", "modified_at"])
+    crawl = Crawl.objects.create(
+        urls=source_text,
+        # The internal root snapshot occupies depth 0. URLs discovered from the
+        # submitted source become normal child snapshots at depth 1, so the
+        # effective crawl limit must be one hop deeper than the user requested.
+        max_depth=depth + 1,
+        tags_str=tag,
+        persona_id=persona_obj.id,
+        label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
+        created_by_id=created_by_id,
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=None if index_only else timezone.now(),
+        config=crawl_config,
+    )
+
+    # Parser plugins consume this root snapshot through the normal Snapshot
+    # hook lifecycle. ArchiveBox does not select parser plugins or call them
+    # directly; non-parser plugins cheaply no-result unsupported internal input.
+    root_snapshot = Snapshot.objects.create(
+        url=Snapshot.INTERNAL_INPUT_URL,
+        crawl=crawl,
+        depth=0,
+        title="stdin.txt",
+    )
+    staticfile_dir = root_snapshot.output_dir / "staticfile"
+    staticfile_dir.mkdir(parents=True, exist_ok=True)
+    (staticfile_dir / "stdin.txt").write_text(source_text, encoding="utf-8")
 
     print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
+    first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
     print(f"    [dim]First URL: {first_url}[/dim]")
 
-    # The runner parses Crawl.urls after claiming the Crawl, then persists the
-    # resulting depth-0 Snapshot facts. Recursive discoveries start at depth 1.
+    # 3. The CrawlMachine will create Snapshots from all URLs when started
+    #    Parser extractors run on snapshots and discover more URLs
+    #    Discovered URLs become child Snapshots (depth+1)
 
     if index_only:
         print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
@@ -219,42 +223,21 @@ def add(
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
-        from archivebox.core.takeover_util import command_owns_foreground_runner, standby_until_foreground_runner_needed
+        from archivebox.machine.models import Process
+        from archivebox.core.takeover_util import command_owns_foreground_runner, current_command, standby_until_foreground_runner_needed
         from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
 
-        assert command is not None
+        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
         exit_code = 0
-        interactive_interrupts = sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
-
-        def interrupt_hook() -> None:
-            raise KeyboardInterrupt
-
-        def crawl_is_complete() -> bool:
-            crawl.refresh_from_db(fields=["status"])
-            return crawl.status not in crawl.RUNNABLE_STATES
-
         try:
             try:
-                with (
-                    foreground_shutdown_signals(
-                        first_signal_message=None,
-                        interrupt_handlers={signal.SIGINT: interrupt_hook},
-                    ),
-                    foreground_parent_watchdog(),
-                ):
+                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
                     while True:
-                        standby = standby_until_foreground_runner_needed(
-                            command,
-                            data_dir=CONSTANTS.DATA_DIR,
-                            work_is_complete=crawl_is_complete,
-                        )
-                        if standby["work_completed"]:
-                            break
+                        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
                         exit_code = run_runner_worker(
                             ["--crawl-id", str(crawl.id)],
                             name=f"worker_runner_add_{os.getpid()}",
-                            interactive_interrupts=interactive_interrupts,
-                            config=get_config(crawl=crawl),
+                            interactive_interrupts=True,
                         )
                         crawl.refresh_from_db(fields=["status", "retry_at"])
                         if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
@@ -318,13 +301,9 @@ def add(
 
             admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
 
-            if snapshots_count:
-                print("\n[bold]snapshot output saved to:[/bold]")
-                for snapshot in crawl.snapshot_set.order_by("created_at").iterator():
-                    click.echo(f"  {snapshot.output_dir}")
-            print("\n[bold]crawl logs and setup:[/bold]")
-            click.echo(f"  {rel_output_str}")
-            click.echo(f"  {admin_url}")
+            print("\n[bold]crawl output saved to:[/bold]")
+            print(f"  {rel_output_str}")
+            print(f"  {admin_url}")
             print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
             print(f"[bold]total size:[/bold] {total_size}")
             print(f"[bold]total time:[/bold] {duration_str}")
@@ -355,7 +334,6 @@ def add(
 @click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
 @click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
 @click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
-@click.option("--extract", default="", help="Comma-separated plugins or output types to extract e.g. title,pdf,text/html,image,...")
 @click.option("--persona", default="Default", help="Authentication profile to use when archiving")
 @click.option(
     "--only-new/--no-only-new",
@@ -405,41 +383,12 @@ def main(**kwargs):
         overwrite = kwargs.pop("overwrite", False)
         update = kwargs.pop("update", False)
         only_new = kwargs.pop("only_new", None)
-        extract = kwargs.pop("extract", "")
         if overwrite or update:
             only_new = False
         if only_new is not None:
             kwargs["config"] = {"ONLY_NEW": bool(only_new)}
-        if extract:
-            from archivebox.plugins.discovery import get_plugin_catalog
 
-            catalog = get_plugin_catalog()
-            tokens = [token.strip() for token in extract.split(",") if token.strip()]
-            plugin_names = {name.lower(): name for name in catalog}
-            selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
-            selected += catalog.matching_output(tokens)
-            if not selected:
-                raise click.UsageError(f"No plugins found matching extract types: {extract}")
-            existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
-            kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
-
-        from archivebox.machine.models import Process
-
-        # CLI commands own their process record and logs. Queue model saves
-        # must not register a process implicitly: HTTP add() shares that path.
-        process = Process.current()
-        exit_code = 0
-        try:
-            add(urls=urls, **kwargs)
-        except BaseException:
-            exit_code = 1
-            raise
-        finally:
-            # --bg/--index-only return without the foreground runner's cleanup.
-            # Leaving them RUNNING makes the active-process index accumulate
-            # historical CLI commands forever, defeating bounded recovery.
-            if process.status == Process.StatusChoices.RUNNING:
-                process.mark_exited(exit_code=exit_code)
+        add(urls=urls, **kwargs)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_archiveresult.py

@@ -6,13 +6,13 @@ archivebox archiveresult <action> [args...] [--filters]
 Manage ArchiveResult records (plugin extraction results).
 
 Actions:
-    create  - Emit plugin extraction request records for Snapshots
+    create  - Create ArchiveResults for Snapshots (queue extractions)
     list    - List ArchiveResults as JSONL (with optional filters)
     update  - Update ArchiveResults from stdin JSONL
     delete  - Delete ArchiveResults from stdin JSONL
 
 Examples:
-    # Emit extraction requests; `archivebox run` schedules their parent snapshots
+    # Create ArchiveResults for snapshots (queue for extraction)
     archivebox snapshot list --status=queued | archivebox archiveresult create
     archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
 
@@ -20,6 +20,9 @@ Examples:
     archivebox archiveresult list --status=failed
     archivebox archiveresult list --plugin=screenshot --status=succeeded
 
+    # Update (reset failed extractions to queued)
+    archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued
+
     # Delete
     archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
 
@@ -141,20 +144,21 @@ def create_archiveresults(
 
     created_count = 0
     for snapshot in snapshots:
-        config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
-        hooks = [
-            hook
-            for hook in discover_hooks("Snapshot", filter_disabled=not plugin, config=config)
-            if not plugin or hook.parent.name == plugin
-        ]
-        for hook_path in hooks:
-            hook_name = hook_path.stem
-            plugin_name = hook_path.parent.name
+        if plugin:
             if not is_tty:
-                write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
+                write_record(build_archiveresult_request(snapshot.id, plugin, status=status))
             created_count += 1
+        else:
+            config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
+            hooks = discover_hooks("Snapshot", config=config)
+            for hook_path in hooks:
+                hook_name = hook_path.stem
+                plugin_name = hook_path.parent.name
+                if not is_tty:
+                    write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
+                created_count += 1
 
-    rprint(f"[green]Created {created_count} extraction request records[/green]", file=sys.stderr)
+    rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
     return 0
 
 
@@ -338,7 +342,7 @@ def main():
 @click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
 @click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
 def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
-    """Emit Snapshot plugin extraction requests as JSONL."""
+    """Create ArchiveResults for Snapshots from stdin JSONL."""
     sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
 
 

archivebox/cli/archivebox_config.py

@@ -59,20 +59,28 @@ def config(
     matching_config = {}
     if search:
         if config_options:
-            search_terms = [key.strip().lower() for key in config_options]
+            config_options = [
+                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
+            ]
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
+            for config_section in CONFIGS.values():
+                aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}
 
-            for existing_key, value in readable_config.items():
-                if any(term in existing_key.lower() or term in str(value).lower() for term in search_terms):
-                    matching_config[existing_key] = value
+                for search_key in config_options:
+                    # search all aliases in the section
+                    for alias_key, key in aliases.items():
+                        if key in readable_config and search_key.lower() in alias_key.lower():
+                            matching_config[key] = dict(config_section)[key]
 
-            for alias, key in core_config_aliases.items():
-                if key in readable_config and any(term in alias.lower() for term in search_terms):
-                    matching_config[key] = readable_config[key]
-
-            for schema in plugin_schemas.values():
-                for key, metadata in schema.items():
-                    if key in readable_config and any(term in key.lower() or term in str(metadata).lower() for term in search_terms):
-                        matching_config[key] = readable_config[key]
+                    # search all keys and values in the section
+                    for existing_key, value in dict(config_section).items():
+                        if existing_key in readable_config and (
+                            search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower()
+                        ):
+                            matching_config[existing_key] = value
+                    for existing_key, value in readonly_config.items():
+                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
+                            matching_config[existing_key] = value
 
         print(printable_config(matching_config))
         raise SystemExit(not matching_config)

archivebox/cli/archivebox_extract.py

@@ -1,135 +1,113 @@
 #!/usr/bin/env python3
 
-"""Run abx-dl snapshot hooks for existing ArchiveBox snapshots."""
+"""
+archivebox extract [snapshot_ids...] [--plugins=NAMES]
+
+Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL.
+
+Input formats:
+    - Snapshot UUIDs (one per line)
+    - JSONL: {"type": "Snapshot", "id": "...", "url": "..."}
+    - JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."}
+
+Output (JSONL):
+    {"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."}
+
+Examples:
+    # Extract specific snapshot
+    archivebox extract 01234567-89ab-cdef-0123-456789abcdef
+
+    # Pipe from snapshot command
+    archivebox snapshot https://example.com | archivebox extract
+
+    # Run specific plugins only
+    archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef
+
+    # Chain commands
+    archivebox crawl https://example.com | archivebox snapshot | archivebox extract
+"""
 
 __package__ = "archivebox.cli"
 __command__ = "archivebox extract"
 
 import sys
 from collections import defaultdict
-from contextlib import redirect_stdout
+from itertools import product
 
 import rich_click as click
 
 
-def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[str]], set[str]]:
-    """Resolve CLI records to snapshot-level execution requests.
-
-    ArchiveResult input is accepted as a convenient reference to its parent
-    Snapshot and plugin. It is never reset or converted into queued work.
+def process_archiveresult_by_id(archiveresult_id: str) -> int:
     """
-    from archivebox.api.v1_core import _uuid_ref_query
-    from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.misc.jsonl import TYPE_ARCHIVERESULT
+    Re-run extraction for a single ArchiveResult by ID.
 
-    explicit_plugins = {name.strip() for name in plugins.split(",") if name.strip()}
-    requested: dict[str, set[str]] = defaultdict(set)
-    missing: set[str] = set()
-
-    for record in records:
-        record_type = record.get("type")
-        record_id = str(record.get("id") or "")
-
-        if record_type == TYPE_ARCHIVERESULT and record.get("snapshot_id"):
-            requested[str(record["snapshot_id"])].update(explicit_plugins or {str(record.get("plugin") or "")})
-            requested[str(record["snapshot_id"])].discard("")
-            continue
-
-        if record_type == TYPE_ARCHIVERESULT:
-            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
-            if result is not None:
-                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
-                continue
-
-        snapshot_id = str(record.get("snapshot_id") or record_id or "")
-        snapshot = Snapshot.objects.filter(id=snapshot_id).only("id").first() if snapshot_id else None
-        if snapshot is None and record.get("url"):
-            snapshot = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").only("id").first()
-        # Bare UUID CLI arguments are parsed as Snapshot records because the
-        # input layer cannot know which model owns them. Preserve the public
-        # convenience of passing an ArchiveResult ID by trying that reference
-        # only after the Snapshot lookup misses.
-        if snapshot is None and record_id and not record.get("url"):
-            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
-            if result is not None:
-                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
-                continue
-        if snapshot is None:
-            missing.add(snapshot_id or str(record.get("url") or ""))
-            continue
-        requested[str(snapshot.id)].update(explicit_plugins)
-
-    return requested, missing
-
-
-def _run_snapshot_requests(requested: dict[str, set[str]], *, wait: bool, show_progress: bool) -> int:
-    from django.utils import timezone
+    ArchiveResults are projected status rows, not queued work items. Re-running
+    a single result means resetting that row and queueing its parent snapshot
+    through the shared crawl runner with the corresponding plugin selected.
+    """
     from rich import print as rprint
-
-    from archivebox.core.models import Snapshot
+    from django.utils import timezone
+    from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.services.runner import run_crawl
 
-    snapshots = {str(snapshot.id): snapshot for snapshot in Snapshot.objects.filter(id__in=requested).select_related("crawl")}
-    if not snapshots:
-        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
+    try:
+        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
+    except ArchiveResult.DoesNotExist:
+        rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
         return 1
 
-    from archivebox.config.common import get_config
-    from archivebox.plugins.discovery import get_enabled_plugins
+    rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
 
-    for snapshot_id, plugin_names in requested.items():
-        snapshot = snapshots.get(snapshot_id)
-        if snapshot is not None and not plugin_names:
-            plugin_names.update(get_enabled_plugins(config=get_config(crawl=snapshot.crawl, snapshot=snapshot)))
+    try:
+        was_paused = archiveresult.snapshot.is_paused
+        archiveresult.reset_for_retry()
+        snapshot = archiveresult.snapshot
+        if not was_paused:
+            snapshot.queue_for_extraction()
+        else:
+            # A paused snapshot may still accept explicit maintenance for one
+            # ArchiveResult, but this path must not transition it back to
+            # queued/startable work. Guard: only set retry_at while the row is
+            # still paused — concurrent resume would otherwise see a stale
+            # retry_at marker.
+            snapshot.safe_update(
+                {"retry_at": timezone.now()},
+                refresh=False,
+                extra_filter={"status": snapshot.StatusChoices.PAUSED},
+            )
+        crawl = snapshot.crawl
+        if not crawl.claim_processing_lock(lock_seconds=10):
+            rprint(
+                f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]",
+                file=sys.stderr,
+            )
+            return 1
 
-    if wait and any("search_backend_sonic" in plugin_names for plugin_names in requested.values()):
-        from archivebox.core.takeover_util import ensure_daemon_stack
+        try:
+            run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        finally:
+            if was_paused:
+                snapshot.restore_paused_scheduler_marker()
+        archiveresult.refresh_from_db()
 
-        with redirect_stdout(sys.stderr):
-            ensure_daemon_stack(reason="Sonic snapshot indexing")
+        if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
+            print(f"[green]Extraction succeeded: {archiveresult.output_str}[/green]")
+            return 0
+        elif archiveresult.status == ArchiveResult.StatusChoices.NORESULTS:
+            print(f"[dim]Extraction completed with no results: {archiveresult.output_str}[/dim]")
+            return 0
+        elif archiveresult.status == ArchiveResult.StatusChoices.FAILED:
+            print(f"[red]Extraction failed: {archiveresult.output_str}[/red]", file=sys.stderr)
+            return 1
+        else:
+            # Still in progress or backoff - not a failure
+            print(f"[yellow]Extraction status: {archiveresult.status}[/yellow]")
+            return 0
 
-    # Explicit extraction resumes open/paused snapshots at the snapshot level.
-    # Sealed snapshots stay sealed during targeted maintenance backfills.
-    if wait:
-        for snapshot in snapshots.values():
-            if snapshot.status != Snapshot.StatusChoices.SEALED:
-                snapshot.update_and_requeue(
-                    status=Snapshot.StatusChoices.QUEUED,
-                    retry_at=timezone.now(),
-                )
-
-    if not wait:
-        for snapshot_id, plugin_names in requested.items():
-            snapshot = snapshots.get(snapshot_id)
-            if snapshot is None:
-                continue
-            if plugin_names:
-                snapshot.schedule_plugin_run(plugin_names, when=timezone.now())
-            elif snapshot.status == Snapshot.StatusChoices.SEALED:
-                snapshot.update_and_requeue(retry_at=timezone.now())
-            else:
-                snapshot.update_and_requeue(
-                    status=Snapshot.StatusChoices.QUEUED,
-                    retry_at=timezone.now(),
-                )
-        if show_progress:
-            rprint(f"[blue]Queued {len(snapshots)} snapshots for extraction[/blue]", file=sys.stderr)
-        return 0
-
-    grouped: dict[tuple[str, tuple[str, ...]], list[str]] = defaultdict(list)
-    for snapshot_id, plugin_names in requested.items():
-        snapshot = snapshots.get(snapshot_id)
-        if snapshot is not None:
-            grouped[(str(snapshot.crawl_id), tuple(sorted(plugin_names)))].append(snapshot_id)
-
-    for (crawl_id, plugin_names), snapshot_ids in grouped.items():
-        run_crawl(
-            crawl_id,
-            snapshot_ids=sorted(snapshot_ids),
-            selected_plugins=list(plugin_names) or None,
-            show_progress=show_progress,
-        )
-    return 0
+    except Exception as e:
+        print(f"[red]Extraction error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+        return 1
 
 
 def run_plugins(
@@ -139,64 +117,353 @@ def run_plugins(
     wait: bool = True,
     emit_results: bool = True,
     show_progress: bool = True,
+    preserve_queued: bool = False,
 ) -> int:
-    """Execute selected plugins through the snapshot-level runner."""
+    """
+    Run plugins on Snapshots from input.
+
+    Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
     from rich import print as rprint
+    from django.utils import timezone
 
+    from archivebox.misc.jsonl import (
+        read_args_or_stdin,
+        write_record,
+        TYPE_SNAPSHOT,
+        TYPE_ARCHIVERESULT,
+    )
     from archivebox.core.models import Snapshot
-    from archivebox.misc.jsonl import read_args_or_stdin, write_record
+    from archivebox.core.models import ArchiveResult
+    from archivebox.services.runner import run_crawl
+    from abx_dl.models import discover_plugins
 
+    is_tty = sys.stdout.isatty()
+
+    # Parse comma-separated plugins list once (reused in creation and filtering)
+    plugins_list = [p.strip() for p in plugins.split(",") if p.strip()] if plugins else []
+
+    # Parse stdin/args exactly once per CLI invocation.
+    # `main()` may already have consumed stdin to distinguish Snapshot input from
+    # ArchiveResult IDs; if so, it must pass the parsed records through here
+    # instead of asking this helper to reread an already-drained pipe.
     if records is None:
         records = list(read_args_or_stdin(args))
+
     if not records:
         rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
         return 1
 
-    requested, missing = _resolve_requests(records, plugins)
-    for value in sorted(missing):
-        rprint(f"[yellow]Snapshot or ArchiveResult not found: {value}[/yellow]", file=sys.stderr)
-    if not requested:
+    # Gather snapshot IDs and optional plugin constraints to process
+    snapshot_ids = set()
+    requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set)
+    for record in records:
+        record_type = record.get("type")
+
+        if record_type == TYPE_SNAPSHOT:
+            snapshot_id = record.get("id")
+            if snapshot_id:
+                snapshot_ids.add(str(snapshot_id))
+            elif record.get("url"):
+                # Look up by URL (get most recent if multiple exist)
+                snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
+                if snap:
+                    snapshot_ids.add(str(snap.id))
+                else:
+                    rprint(f"[yellow]Snapshot not found for URL: {record['url']}[/yellow]", file=sys.stderr)
+
+        elif record_type == TYPE_ARCHIVERESULT:
+            snapshot_id = record.get("snapshot_id")
+            if snapshot_id:
+                snapshot_ids.add(str(snapshot_id))
+                plugin_name = record.get("plugin")
+                if plugin_name and not plugins_list:
+                    requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
+
+        elif "id" in record:
+            # Assume it's a snapshot ID
+            snapshot_ids.add(str(record["id"]))
+
+    if not snapshot_ids:
+        rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
         return 1
 
-    exit_code = _run_snapshot_requests(requested, wait=wait, show_progress=show_progress)
-    if exit_code or not emit_results:
-        return exit_code
+    existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id"))
+    existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots}
+    existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots}
+    missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids)
+    for snapshot_id in missing_snapshot_ids:
+        rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
+
+    # Queue only the target plugin rows. Bulk updates keep large reindex runs
+    # from doing one SELECT+UPDATE per snapshot/plugin before hooks even start.
+    requested_pairs: set[tuple[str, str]] = set()
+    if plugins_list:
+        requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list))
+    else:
+        requested_pairs.update(
+            (snapshot_id, plugin_name)
+            for snapshot_id, plugin_names in requested_plugins_by_snapshot.items()
+            if snapshot_id in existing_snapshot_ids
+            for plugin_name in plugin_names
+        )
+    plugins_by_name = discover_plugins(runtime="archivebox")
+    requested_rows: set[tuple[str, str, str]] = set()
+    for snapshot_id, plugin_name in requested_pairs:
+        plugin = plugins_by_name.get(plugin_name)
+        hooks = plugin.filter_hooks("Snapshot") if plugin is not None else []
+        if hooks:
+            requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks)
+        else:
+            requested_rows.add((snapshot_id, plugin_name, ""))
+
+    queued_rows: set[tuple[str, str, str]] = set()
+    if preserve_queued and requested_rows:
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+    rows_to_queue = requested_rows - queued_rows
+
+    reset_fields = {
+        "status": ArchiveResult.StatusChoices.QUEUED,
+        "output_str": "",
+        "output_json": None,
+        "output_files": {},
+        "output_size": 0,
+        "output_mimetypes": "",
+        "start_ts": None,
+        "end_ts": None,
+        "modified_at": timezone.now(),
+    }
+    if rows_to_queue and plugins_list:
+        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    elif rows_to_queue and requested_plugins_by_snapshot:
+        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    existing_rows = (
+        {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        if rows_to_queue
+        else set()
+    )
+    missing_rows = rows_to_queue - existing_rows
+    if missing_rows:
+        ArchiveResult.objects.bulk_create(
+            [
+                ArchiveResult(
+                    snapshot_id=snapshot_id,
+                    plugin=plugin_name,
+                    hook_name=hook_name,
+                    status=ArchiveResult.StatusChoices.QUEUED,
+                )
+                for snapshot_id, plugin_name, hook_name in sorted(missing_rows)
+            ],
+            batch_size=500,
+        )
+
+    processed_count = len(existing_snapshot_ids)
+    queue_at = timezone.now()
+    if existing_snapshot_ids:
+        if requested_rows:
+            # Targeted ArchiveResult retries use retry_at as the scheduling
+            # signal and keep sealed snapshots sealed so extractors are not
+            # re-run outside the explicitly queued plugin rows. Paused snapshots
+            # also keep status=paused here: `retry_at` only asks the orchestrator
+            # to process the queued plugin rows, and run_due_snapshot restores
+            # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
+            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
+            if preserve_queued and queued_rows:
+                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
+                    .filter(retry_at__gt=queue_at)
+                    .values_list("id", flat=True)
+                )
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
+                        "id",
+                        flat=True,
+                    )
+                )
+            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
+                # Guard the read-time status so we never bump retry_at on a
+                # row that's been re-queued / started by a concurrent runner.
+                snapshot.safe_update(
+                    {"retry_at": queue_at, "modified_at": queue_at},
+                    refresh=False,
+                    extra_filter={"status": snapshot.status},
+                )
+        else:
+            # No plugin rows were requested, so this is a full snapshot retry.
+            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):
+                snapshot.safe_update(
+                    {
+                        "status": Snapshot.StatusChoices.QUEUED,
+                        "retry_at": queue_at,
+                        "current_step": 0,
+                        "modified_at": queue_at,
+                    },
+                    refresh=False,
+                    extra_filter={"status": snapshot.status},
+                )
+    if existing_crawl_ids and not requested_rows:
+        from archivebox.crawls.models import Crawl
+
+        for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"):
+            update_fields = {
+                "retry_at": queue_at,
+                "modified_at": queue_at,
+            }
+            if crawl.status != Crawl.StatusChoices.STARTED:
+                update_fields["status"] = Crawl.StatusChoices.QUEUED
+            crawl.safe_update(
+                update_fields,
+                refresh=False,
+                extra_filter={"status": crawl.status},
+            )
+
+    if processed_count == 0:
+        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
+        return 1
+
+    if show_progress:
+        rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
+
+    # Run orchestrator if --wait (default)
+    if wait:
+        if show_progress:
+            rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
+        snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
+        for snapshot_id, crawl_id in existing_snapshots:
+            snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id))
+
+        for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
+            from archivebox.crawls.models import Crawl
+
+            crawl = Crawl.objects.get(id=crawl_id)
+            if not crawl.claim_processing_lock(lock_seconds=10):
+                rprint(
+                    f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]",
+                    file=sys.stderr,
+                )
+                return 1
+            selected_plugins = (
+                plugins_list
+                or sorted(
+                    {plugin for snapshot_id in crawl_snapshot_ids for plugin in requested_plugins_by_snapshot.get(str(snapshot_id), set())},
+                )
+                or None
+            )
+            run_crawl(
+                crawl_id,
+                snapshot_ids=sorted(crawl_snapshot_ids),
+                selected_plugins=selected_plugins,
+                show_progress=show_progress,
+            )
+
+    if not emit_results:
+        return 0
+
+    # Output results as JSONL (when piped) or human-readable (when TTY)
+    for snapshot_id in snapshot_ids:
+        try:
+            snapshot = Snapshot.objects.get(id=snapshot_id)
+            results = snapshot.archiveresult_set.all()
+            if plugins_list:
+                results = results.filter(plugin__in=plugins_list)
+
+            for result in results:
+                if is_tty:
+                    status_color = {
+                        "succeeded": "green",
+                        "failed": "red",
+                        "skipped": "yellow",
+                    }.get(result.status, "dim")
+                    rprint(
+                        f"  [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ''}",
+                        file=sys.stderr,
+                    )
+                else:
+                    write_record(result.to_json())
+        except Snapshot.DoesNotExist:
+            continue
 
-    is_tty = sys.stdout.isatty()
-    for snapshot in Snapshot.objects.filter(id__in=requested):
-        results = snapshot.archiveresult_set.all()
-        requested_plugins = requested[str(snapshot.id)]
-        if requested_plugins:
-            results = results.filter(plugin__in=requested_plugins)
-        for result in results:
-            if is_tty:
-                color = {"succeeded": "green", "failed": "red", "skipped": "yellow"}.get(result.status, "dim")
-                rprint(f"  [{color}]{result.status}[/{color}] {result.plugin} → {result.output_str or ''}", file=sys.stderr)
-            else:
-                write_record(result.to_json())
     return 0
 
 
-def process_archiveresult_by_id(archiveresult_id: str) -> int:
-    """Re-run the parent Snapshot plugin referenced by an ArchiveResult."""
-    return run_plugins((), records=[{"id": archiveresult_id}], wait=True)
+def is_archiveresult_id(value: str) -> bool:
+    """Check if value resolves to an ArchiveResult ID."""
+    from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
+
+    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()
 
 
 @click.command()
-@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run")
+@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run (e.g., screenshot,singlefile)")
 @click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
 @click.argument("args", nargs=-1)
 def main(plugins: str, wait: bool, args: tuple):
-    """Run plugins on Snapshots; ArchiveResult IDs select their parent plugin."""
+    """Run plugins on Snapshots, or process existing ArchiveResults by ID"""
     from archivebox.misc.jsonl import read_args_or_stdin
 
+    # Read all input
     records = list(read_args_or_stdin(args))
+
     if not records:
         from rich import print as rprint
 
         rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
         sys.exit(1)
-    sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
+
+    # Check if input looks like existing ArchiveResult IDs to process
+    all_are_archiveresult_ids = all(is_archiveresult_id(r.get("id") or r.get("url", "")) for r in records)
+
+    if all_are_archiveresult_ids:
+        # Process existing ArchiveResults by ID
+        from rich import print as rprint
+
+        exit_code = 0
+        for record in records:
+            archiveresult_id = record.get("id") or record.get("url")
+            if not isinstance(archiveresult_id, str):
+                rprint(f"[red]Invalid ArchiveResult input: {record}[/red]", file=sys.stderr)
+                exit_code = 1
+                continue
+            result = process_archiveresult_by_id(archiveresult_id)
+            if result != 0:
+                exit_code = result
+        sys.exit(exit_code)
+    else:
+        # Default behavior: run plugins on Snapshots from input
+        sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_help.py

@@ -5,8 +5,6 @@ __command__ = "archivebox help"
 import ast
 import importlib.util
 import os
-import sys
-from datetime import UTC, datetime
 from pathlib import Path
 
 import click
@@ -42,14 +40,11 @@ def help() -> None:
 
     from archivebox.cli import ArchiveBoxGroup
     from archivebox.config import CONSTANTS
+    from archivebox.config.common import get_config
     from archivebox.config.permissions import IN_DOCKER
+    from archivebox.misc.logging_util import log_cli_command
 
-    version_msg = (
-        f"[dark_magenta]\\[{datetime.now(UTC):%Y-%m-%d %H:%M:%S}][/dark_magenta] "
-        f"[dark_red]ArchiveBox[/dark_red] [dark_goldenrod]v{CONSTANTS.VERSION}[/dark_goldenrod]: "
-        "[green4]archivebox [green3]help"
-    )
-    print(Panel(version_msg), file=sys.stderr)
+    log_cli_command("help", [], None, ".")
 
     COMMANDS_HELP_TEXT = (
         "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.meta_commands.items())
@@ -68,7 +63,7 @@ def help() -> None:
     [blue]docker compose run[/blue] [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
 
     [grey53]# using Docker:[/grey53]
-    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 5797:5797[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
+    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 8000:8000[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
 """
         if IN_DOCKER
         else ""
@@ -94,6 +89,7 @@ def help() -> None:
     [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
 """)
 
+    get_config()
     if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
         pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
         EXAMPLE_USAGE = f"""
@@ -109,7 +105,7 @@ def help() -> None:
     [dark_green]archivebox[/dark_green] [green]add[/green] --depth=1 "https://example.com/some/page"
     [dark_green]archivebox[/dark_green] [green]list[/green] --sort=timestamp --csv=timestamp,downloaded_at,url,title
     [dark_green]archivebox[/dark_green] [green]schedule[/green] --every=day --depth=1 "https://example.com/some/feed.rss"
-    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:5797[/blue]                [grey53]# Start the Web UI / API server[/grey53]
+    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:8000[/blue]                [grey53]# Start the Web UI / API server[/grey53]
 """
         print(
             Panel(

archivebox/cli/archivebox_init.py

@@ -25,20 +25,20 @@ def _display_data_path(path: Path, data_dir: Path) -> str:
 def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
     """Initialize a new ArchiveBox collection in the current directory"""
 
-    from archivebox.misc.checks import check_not_inside_source_dir
-
-    check_not_inside_source_dir()
-
     from archivebox.config import CONSTANTS, VERSION
     from archivebox.config.common import get_config
     from archivebox.config.collection import write_config_file
-    from archivebox.misc.db import apply_migrations, database_exists, ensure_database_ready
+    from archivebox.misc.db import apply_migrations
     from archivebox.misc.checks import check_migrations
 
     config = get_config()
 
+    # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
+    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
+    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
+
     is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
-    existing_index = database_exists()
+    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
     if is_empty and not existing_index:
         print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
         print("[green]----------------------------------------------------------------------[/green]")
@@ -86,15 +86,11 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     # create the ArchiveBox.conf file
     write_config_file({"SECRET_KEY": config.SECRET_KEY})
 
-    if existing_index:
+    if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
         print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]")
     else:
         print("\n[green][+] Building main SQL index and running initial migrations...[/green]")
 
-    # For postgres, make sure the server is reachable and create the database
-    # if it doesn't exist yet (sqlite creates its file automatically).
-    ensure_database_ready()
-
     from archivebox.config.django import setup_django
 
     setup_django()
@@ -111,24 +107,12 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
         else:
             os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
 
-    from archivebox.machine.models import Machine
-
-    # Registration requires the final schema and reconciles legacy file config.
-    Machine.current()
-
-    from archivebox.personas.models import Persona
-
-    # Initialize here, not when /add/ is rendered: a GET must not create profiles.
-    Persona.get_or_create_default().ensure_dirs()
-
-    from archivebox.misc.db import database_display_location, is_postgres
-
-    assert database_exists()
+    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
     print()
-    if is_postgres():
-        print(f"    √ {database_display_location()}")
-    else:
-        print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
+    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
+
+    # from django.contrib.auth.models import User
+    #     call_command("createsuperuser", interactive=True)
 
     print()
     print("[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]")
@@ -144,7 +128,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     print("    > Skipping orphan snapshot import during init.")
     print()
     print("    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
-    print("        archivebox update --rescan --migrate-only")
+    print("        archivebox update")
 
     print("\n[green]----------------------------------------------------------------------[/green]")
 
@@ -165,12 +149,12 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
     CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
 
-    from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir
+    from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir
 
     config = get_config()
-    create_and_chown_dir(config.TMP_DIR)
-    create_and_chown_dir(config.ABXPKG_LIB_DIR)
-    create_and_chown_dir(config.ABXPKG_LIB_DIR / "bin")
+    config.TMP_DIR.mkdir(parents=True, exist_ok=True)
+    config.LIB_DIR.mkdir(parents=True, exist_ok=True)
+    (config.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
 
     working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
     if working_tmp_dir:
@@ -186,13 +170,10 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
         install_method()
 
     if Snapshot.objects.count() < 25:  # hide the hints for experienced users
-        from archivebox.core.routes_util import build_admin_url
-
-        admin_url = build_admin_url("/admin/", config=config)
         print()
-        print("    [violet]Hint:[/violet] To finish web setup, run:")
+        print("    [violet]Hint:[/violet] To view your archive index, run:")
         print(
-            f"        archivebox server  # then visit [deep_sky_blue4][link={admin_url}]{admin_url}[/link][/deep_sky_blue4]",
+            "        archivebox server  # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]",
         )
         print()
         print("    To add new links, you can run:")
@@ -205,7 +186,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
 @click.command()
 @click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
 @click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
-@click.option("--install", "--setup", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
+@click.option("--install", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
 @docstring(init.__doc__)
 def main(**kwargs) -> None:
     init(**kwargs)

archivebox/cli/archivebox_install.py

@@ -3,9 +3,6 @@
 __package__ = "archivebox.cli"
 
 import os
-import shutil
-import re
-from pathlib import Path
 
 import rich_click as click
 from rich import print
@@ -13,113 +10,6 @@ from rich import print
 from archivebox.misc.util import docstring, enforce_types
 
 
-def _resolve_install_targets(
-    requested_names: tuple[str, ...],
-) -> tuple[list[str], list[str]]:
-    """Resolve plugin names and declared binary aliases, leaving unknown tokens raw."""
-    from archivebox.plugins.discovery import get_plugin_catalog
-
-    catalog = get_plugin_catalog()
-    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in catalog}
-    plugin_names_by_binary_alias: dict[str, set[str]] = {}
-    for plugin_name, plugin in catalog.items():
-        for required_binary in plugin.config.required_binaries:
-            aliases = {required_binary.name.lower()}
-            template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", required_binary.name)
-            if template_match:
-                config_key = template_match.group(1)
-                aliases.add(config_key.lower())
-                aliases.add(config_key.removesuffix("_BINARY").lower())
-                default = plugin.config.properties.get(config_key, {}).get("default")
-                if default:
-                    aliases.add(str(default).lower())
-            for alias in aliases:
-                plugin_names_by_binary_alias.setdefault(alias, set()).add(plugin_name)
-
-    requested_plugins: list[str] = []
-    raw_binary_names: list[str] = []
-    for name in requested_names:
-        normalized_name = name.lower()
-        if normalized_name in plugin_names_by_lower:
-            requested_plugins.append(plugin_names_by_lower[normalized_name])
-        elif normalized_name in plugin_names_by_binary_alias:
-            requested_plugins.extend(sorted(plugin_names_by_binary_alias[normalized_name]))
-        else:
-            raw_binary_names.append(name)
-
-    selected_plugins = catalog.select(requested_plugins) if requested_plugins else catalog.select([])
-    selected_plugin_names = {name.lower() for name in selected_plugins}
-    raw_binary_names = [name for name in raw_binary_names if name.lower() not in selected_plugin_names]
-    return sorted(selected_plugins), sorted(set(raw_binary_names))
-
-
-def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> None:
-    """Install user-requested standalone binaries through the Binary lifecycle."""
-    from django.utils import timezone
-
-    from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-
-    machine = Machine.current()
-    for requested_name in binary_names:
-        binary_name = _canonical_binary_name(requested_name)
-        if not binary_name:
-            raise click.UsageError(f"Invalid binary name: {requested_name}")
-        binary = Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").first()
-        if binary is None:
-            binary = Binary.objects.create(
-                machine=machine,
-                name=binary_name,
-                binproviders=binproviders,
-                overrides={},
-                status=Binary.StatusChoices.QUEUED,
-            )
-        elif not binary.is_valid or binary.binproviders != binproviders:
-            binary.binproviders = binproviders
-            binary.overrides = {}
-            binary.status = Binary.StatusChoices.QUEUED
-            binary.retry_at = timezone.now()
-            binary.save(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
-        if binary.status == Binary.StatusChoices.INSTALLED:
-            continue
-        # The synchronous CLI needs the lifecycle's verified installation result,
-        # not the scheduler's "claimed work" result. Reuse its failure/requeue and
-        # health-stat handling so a failed install cannot exit successfully.
-        if not binary.install_claimed(lock_seconds=60):
-            raise click.ClickException(f"Binary {binary_name} is already being installed")
-
-
-def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:
-    """Expose the shared abxpkg lib dir at DATA_DIR/lib after install."""
-    link_path = data_dir / "lib"
-    target_config_path = abxpkg_lib_dir.expanduser().absolute()
-    target_path = abxpkg_lib_dir.expanduser().resolve()
-    from archivebox.config.paths import get_machine_type
-
-    scoped_data_lib_path = (link_path / get_machine_type()).absolute()
-
-    if target_config_path == scoped_data_lib_path:
-        if link_path.is_symlink() or link_path.is_file():
-            link_path.unlink()
-        target_config_path.mkdir(parents=True, exist_ok=True)
-        return None
-
-    if link_path.resolve(strict=False) == target_path:
-        return None
-
-    target_path.mkdir(parents=True, exist_ok=True)
-
-    if link_path.is_symlink():
-        if link_path.resolve(strict=False) == target_path:
-            return link_path
-        link_path.unlink()
-    elif link_path.exists():
-        shutil.rmtree(link_path)
-
-    relative_target = os.path.relpath(target_path, start=link_path.parent)
-    link_path.symlink_to(relative_target, target_is_directory=True)
-    return link_path
-
-
 @enforce_types
 def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
     """Detect and install ArchiveBox dependencies by running the abx-dl install flow
@@ -166,32 +56,16 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
 
     setup_django()
 
-    plugin_names, raw_binary_names = _resolve_install_targets(binaries) if binaries else ([], [])
-    install_plugin_names = list(plugin_names)
+    plugin_names = list(binaries)
     if binproviders != "*":
-        install_plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
+        plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
 
-    if install_plugin_names or not binaries:
-        if not binaries:
-            from archivebox.workers.supervisord_util import require_dependency_install_memory
+    print("[+] Running installer via abx-dl bus...")
+    print()
 
-            require_dependency_install_memory()
-        print("[+] Running plugin installer via abx-dl bus...")
-        print()
+    from archivebox.services.runner import run_install
 
-        from archivebox.services.runner import run_install
-
-        run_install(plugin_names=install_plugin_names or None)
-
-    if raw_binary_names:
-        print(f"[+] Running direct binary installer via ArchiveBox binary lifecycle: {', '.join(raw_binary_names)}")
-        print()
-        _install_raw_binary_names(raw_binary_names, binproviders)
-
-    from archivebox.config.common import get_config
-
-    config = get_config(include_machine=False)
-    ensure_data_dir_lib_symlink(CONSTANTS.DATA_DIR, config.ABXPKG_LIB_DIR)
+    run_install(plugin_names=plugin_names or None)
 
     print()
 
@@ -201,10 +75,8 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
     User = get_user_model()
 
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
-        from archivebox.core.routes_util import build_admin_url
-
-        stderr("\n[+] Open the Admin UI to create the first admin and finish web setup:", color="green")
-        stderr(f"    {build_admin_url('/admin/', config=get_config(resolve_plugins=False))}")
+        stderr("\n[+] Don't forget to create a new admin user for the Web UI...", color="green")
+        stderr("    archivebox manage createsuperuser")
 
     print()
 
@@ -212,7 +84,7 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
     # Django is already loaded, so just import and call the function directly
     from archivebox.cli.archivebox_version import version as show_version
 
-    show_version(quiet=False, binaries=binaries)
+    show_version(quiet=False)
 
 
 @click.command()

archivebox/cli/archivebox_manage.py

@@ -14,7 +14,7 @@ def manage(args: list[str] | None = None) -> None:
     from archivebox.misc.logging import stderr
 
     config = get_config()
-    if (args and "createsuperuser" in args and "--noinput" not in args) and (config.IN_DOCKER and not config.IS_TTY):
+    if (args and "createsuperuser" in args) and (config.IN_DOCKER and not config.IS_TTY):
         stderr("[!] Warning: you need to pass -it to use interactive commands in docker", color="lightyellow")
         stderr("    docker run -it archivebox manage {}".format(" ".join(args or ["..."])), color="lightyellow")
         stderr("")

archivebox/cli/archivebox_oneshot.py

@@ -1,49 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox oneshot"
-
-import subprocess
-import sys
-from pathlib import Path
-
-import rich_click as click
-
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
-
-
-@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
-@click.argument("args", nargs=-1)
-def main(args: tuple[str, ...] = ()) -> None:
-    """Download URLs using abx-dl"""
-    from archivebox.misc.checks import check_not_inside_source_dir
-
-    check_not_inside_source_dir()
-
-    cwd = Path.cwd()
-    if any((path / CONSTANTS.SQL_INDEX_FILENAME).exists() for path in (cwd, *cwd.parents)):
-        raise click.ClickException(
-            "Refusing to run `archivebox oneshot` inside an ArchiveBox DATA_DIR. Use `archivebox add` here, or run oneshot from another directory.",
-        )
-    abxpkg_binary = Path(sys.executable).with_name("abxpkg")
-    if not abxpkg_binary.is_file():
-        raise click.ClickException(f"abxpkg executable is missing from the ArchiveBox environment: {abxpkg_binary}")
-    abxpkg_lib_dir = get_config(include_machine=False).ABXPKG_LIB_DIR
-    raise SystemExit(
-        subprocess.run(
-            [
-                str(abxpkg_binary),
-                f"--lib={abxpkg_lib_dir}",
-                "--binproviders=env",
-                "--install",
-                "run",
-                "abx-dl",
-                *args,
-            ],
-        ).returncode,
-    )
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_persona.py

@@ -27,10 +27,11 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox persona"
 
+import os
 import sys
 import shutil
+import platform
 from pathlib import Path
-from dataclasses import replace
 from collections.abc import Iterable
 
 import rich_click as click
@@ -40,6 +41,175 @@ from archivebox.cli.cli_util import apply_filters
 from archivebox.personas import importers as persona_importers
 
 
+# =============================================================================
+# Browser Profile Locations
+# =============================================================================
+
+
+def get_chrome_user_data_dir() -> Path | None:
+    """Get the default Chrome user data directory for the current platform."""
+    system = platform.system()
+    home = Path.home()
+
+    if system == "Darwin":  # macOS
+        candidates = [
+            home / "Library" / "Application Support" / "Google" / "Chrome",
+            home / "Library" / "Application Support" / "Chromium",
+        ]
+    elif system == "Linux":
+        candidates = [
+            home / ".config" / "google-chrome",
+            home / ".config" / "chromium",
+            home / ".config" / "chrome",
+            home / "snap" / "chromium" / "common" / "chromium",
+        ]
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = [
+            local_app_data / "Google" / "Chrome" / "User Data",
+            local_app_data / "Chromium" / "User Data",
+        ]
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate.exists() and (candidate / "Default").exists():
+            return candidate
+
+    return None
+
+
+def get_brave_user_data_dir() -> Path | None:
+    """Get the default Brave user data directory for the current platform."""
+    system = platform.system()
+    home = Path.home()
+
+    if system == "Darwin":
+        candidates = [
+            home / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser",
+        ]
+    elif system == "Linux":
+        candidates = [
+            home / ".config" / "BraveSoftware" / "Brave-Browser",
+        ]
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = [
+            local_app_data / "BraveSoftware" / "Brave-Browser" / "User Data",
+        ]
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate.exists() and (candidate / "Default").exists():
+            return candidate
+
+    return None
+
+
+def get_edge_user_data_dir() -> Path | None:
+    """Get the default Edge user data directory for the current platform."""
+    system = platform.system()
+    home = Path.home()
+
+    if system == "Darwin":
+        candidates = [
+            home / "Library" / "Application Support" / "Microsoft Edge",
+        ]
+    elif system == "Linux":
+        candidates = [
+            home / ".config" / "microsoft-edge",
+            home / ".config" / "microsoft-edge-beta",
+            home / ".config" / "microsoft-edge-dev",
+        ]
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = [
+            local_app_data / "Microsoft" / "Edge" / "User Data",
+        ]
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate.exists() and (candidate / "Default").exists():
+            return candidate
+
+    return None
+
+
+def get_browser_binary(browser: str) -> str | None:
+    system = platform.system()
+    home = Path.home()
+    browser = browser.lower()
+
+    if system == "Darwin":
+        candidates = {
+            "chrome": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"],
+            "chromium": ["/Applications/Chromium.app/Contents/MacOS/Chromium"],
+            "brave": ["/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"],
+            "edge": ["/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"],
+        }.get(browser, [])
+    elif system == "Linux":
+        candidates = {
+            "chrome": [
+                "/usr/bin/google-chrome",
+                "/usr/bin/google-chrome-stable",
+                "/usr/bin/google-chrome-beta",
+                "/usr/bin/google-chrome-unstable",
+            ],
+            "chromium": ["/usr/bin/chromium", "/usr/bin/chromium-browser"],
+            "brave": ["/usr/bin/brave-browser", "/usr/bin/brave-browser-beta", "/usr/bin/brave-browser-nightly"],
+            "edge": [
+                "/usr/bin/microsoft-edge",
+                "/usr/bin/microsoft-edge-stable",
+                "/usr/bin/microsoft-edge-beta",
+                "/usr/bin/microsoft-edge-dev",
+            ],
+        }.get(browser, [])
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = {
+            "chrome": [
+                str(local_app_data / "Google" / "Chrome" / "Application" / "chrome.exe"),
+                "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
+                "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
+            ],
+            "chromium": [str(local_app_data / "Chromium" / "Application" / "chrome.exe")],
+            "brave": [
+                str(local_app_data / "BraveSoftware" / "Brave-Browser" / "Application" / "brave.exe"),
+                "C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
+                "C:\\Program Files (x86)\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
+            ],
+            "edge": [
+                str(local_app_data / "Microsoft" / "Edge" / "Application" / "msedge.exe"),
+                "C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
+                "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
+            ],
+        }.get(browser, [])
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate and Path(candidate).exists():
+            return candidate
+
+    return None
+
+
+BROWSER_PROFILE_FINDERS = {
+    "chrome": get_chrome_user_data_dir,
+    "chromium": get_chrome_user_data_dir,  # Same locations
+    "brave": get_brave_user_data_dir,
+    "edge": get_edge_user_data_dir,
+}
+
+CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"}
+
+
+# =============================================================================
+# Cookie Extraction via CDP
+# =============================================================================
+
 # =============================================================================
 # Validation Helpers
 # =============================================================================
@@ -107,9 +277,6 @@ def create_personas(
     names: Iterable[str],
     import_from: str | None = None,
     profile: str | None = None,
-    source: str | None = None,
-    browser_binary: str | None = None,
-    permissions: str | None = None,
 ) -> int:
     """
     Create Personas from names.
@@ -131,26 +298,29 @@ def create_personas(
         rprint("[yellow]No persona names provided. Pass names as arguments.[/yellow]", file=sys.stderr)
         return 1
 
-    import_source = None
-    if source and not import_from:
-        rprint("[red]--source requires --import (the source browser name).[/red]", file=sys.stderr)
-        return 1
+    # Validate import source if specified
+    source_profile_dir = None
     if import_from:
-        try:
-            if source:
-                import_source = persona_importers.resolve_custom_import_source(source, profile_dir=profile)
-                if import_source.browser != "persona":
-                    import_source = replace(import_source, browser=import_from.lower(), browser_binary=browser_binary)
-            elif import_from.startswith(("http://", "https://", "ws://", "wss://")) or Path(import_from).expanduser().is_absolute():
-                import_source = persona_importers.resolve_custom_import_source(import_from)
-            else:
-                import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile)
-            if browser_binary:
-                import_source = replace(import_source, browser_binary=browser_binary)
-        except ValueError as err:
-            rprint(f"[red]{err}[/red]", file=sys.stderr)
+        import_from = import_from.lower()
+        if import_from not in BROWSER_PROFILE_FINDERS:
+            rprint(f"[red]Unknown browser: {import_from}[/red]", file=sys.stderr)
+            rprint(f"[dim]Supported browsers: {', '.join(BROWSER_PROFILE_FINDERS.keys())}[/dim]", file=sys.stderr)
             return 1
 
+        source_profile_dir = BROWSER_PROFILE_FINDERS[import_from]()
+        if not source_profile_dir:
+            rprint(f"[red]Could not find {import_from} profile directory[/red]", file=sys.stderr)
+            return 1
+
+        rprint(f"[dim]Found {import_from} profile: {source_profile_dir}[/dim]", file=sys.stderr)
+
+        if profile is None and (source_profile_dir / "Default").exists():
+            profile = "Default"
+
+        browser_binary = get_browser_binary(import_from)
+        if browser_binary:
+            rprint(f"[dim]Using {import_from} binary: {browser_binary}[/dim]", file=sys.stderr)
+
     created_count = 0
     for name in name_list:
         name = name.strip()
@@ -163,25 +333,21 @@ def create_personas(
             rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
             continue
 
-        created = Persona.find_named(name) is None
-        persona = Persona.get_or_create_named(
-            name,
-            defaults={"config": {"PERMISSIONS": permissions}} if permissions else {},
-        )
+        persona, created = Persona.objects.get_or_create(name=name)
 
         if created:
             persona.ensure_dirs()
             created_count += 1
-
+            rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
         else:
             rprint(f"[dim]Persona already exists: {name}[/dim]", file=sys.stderr)
 
         cookies_file = Path(persona.path) / "cookies.txt"
 
         # Import browser profile if requested
-        if import_source is not None:
+        if import_from in CHROMIUM_BROWSERS and source_profile_dir is not None:
             try:
-                rprint(f"[dim]Importing {import_source.display_label}; copying settings and exporting cookies...[/dim]", file=sys.stderr)
+                import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile)
                 import_result = persona_importers.import_persona_from_source(
                     persona,
                     import_source,
@@ -189,25 +355,20 @@ def create_personas(
                     import_cookies=True,
                     capture_storage=False,
                 )
-            except (Exception, KeyboardInterrupt) as e:
-                if created:
-                    shutil.rmtree(persona.path, ignore_errors=True)
-                    persona.delete()
+            except Exception as e:
                 rprint(f"[red]Failed to import browser profile: {e}[/red]", file=sys.stderr)
                 return 1
 
             if import_result.profile_copied:
                 rprint("[green]Copied browser profile to persona[/green]", file=sys.stderr)
             if import_result.cookies_imported:
-                rprint(f"[green]Extracted {import_result.cookie_count} cookies to {cookies_file}[/green]", file=sys.stderr)
+                rprint(f"[green]Extracted cookies to {cookies_file}[/green]", file=sys.stderr)
             elif not import_result.profile_copied:
                 rprint("[yellow]Could not import cookies automatically.[/yellow]", file=sys.stderr)
 
             for warning in import_result.warnings:
                 rprint(f"[yellow]{warning}[/yellow]", file=sys.stderr)
 
-        if created:
-            rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
         if not is_tty:
             write_record(
                 {
@@ -314,9 +475,7 @@ def update_personas(name: str | None = None) -> int:
             if persona_id:
                 persona = Persona.objects.get(id=persona_id)
             else:
-                persona = Persona.find_named(old_name)
-                if persona is None:
-                    raise Persona.DoesNotExist
+                persona = Persona.objects.get(name=old_name)
 
             # Apply updates from CLI flags
             if name:
@@ -328,7 +487,7 @@ def update_personas(name: str | None = None) -> int:
 
                 # Rename the persona directory too
                 old_path = persona.path
-                persona.name = persona.validate_name(name)
+                persona.name = name
                 new_path = persona.path
 
                 if old_path.exists() and old_path != new_path:
@@ -448,42 +607,11 @@ def main():
 
 @main.command("create")
 @click.argument("names", nargs=-1)
-@click.option(
-    "--import",
-    "import_from",
-    help="Import from chrome, chrome-beta, chromium, brave, edge, a persona directory, or a live CDP URL",
-)
-@click.option("--source", help="Source browser user-data directory or exact profile path (including Docker mounts)")
-@click.option(
-    "--browser-binary",
-    type=click.Path(exists=True, dir_okay=False),
-    help="Source Chromium browser executable; required for other Chromium-based browsers",
-)
+@click.option("--import", "import_from", help="Import profile from browser (chrome, chromium, brave, edge)")
 @click.option("--profile", help="Profile directory name under the user data dir (e.g. Default, Profile 1)")
-@click.option(
-    "--permissions",
-    type=click.Choice(["public", "unlisted", "private"]),
-    help="Visibility for a newly created persona (existing personas are unchanged)",
-)
-def create_cmd(
-    names: tuple,
-    import_from: str | None,
-    profile: str | None,
-    source: str | None,
-    browser_binary: str | None,
-    permissions: str | None,
-):
+def create_cmd(names: tuple, import_from: str | None, profile: str | None):
     """Create Personas, optionally importing from a browser profile."""
-    sys.exit(
-        create_personas(
-            names,
-            import_from=import_from,
-            profile=profile,
-            source=source,
-            browser_binary=browser_binary,
-            permissions=permissions,
-        ),
-    )
+    sys.exit(create_personas(names, import_from=import_from, profile=profile))
 
 
 @main.command("list")
@@ -495,84 +623,6 @@ def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
     sys.exit(list_personas(name=name, name__icontains=name__icontains, limit=limit))
 
 
-@main.command("open")
-@click.argument("name", default="Default")
-def open_cmd(name: str):
-    """Open a persona's browser, including its imported logins."""
-    import json
-    import os
-    import subprocess
-
-    import psutil
-
-    from abx_plugins import get_plugins_dir
-    from archivebox.config.common import get_config
-    from archivebox.personas.models import Persona
-
-    valid, reason = validate_persona_name(name)
-    if not valid:
-        raise click.ClickException(reason)
-    persona = Persona.get_or_create_named(name)
-    persona.ensure_dirs()
-    config = get_config(persona=persona)
-    if os.environ.get("DISPLAY"):
-        from archivebox.config import CONSTANTS
-
-        for process in psutil.process_iter():
-            try:
-                args = process.cmdline()
-                profile = next((arg.split("=", 1)[1] for arg in args if arg.startswith("--user-data-dir=")), None)
-                if (
-                    not profile
-                    or any(arg.startswith(("--type=", "--headless")) for arg in args)
-                    or not Path(profile).resolve().is_relative_to(CONSTANTS.DATA_DIR.resolve())
-                    or process.environ().get("DISPLAY") != os.environ["DISPLAY"]
-                ):
-                    continue
-                rprint("An ArchiveBox browser is already open on this display; waiting for it to close.")
-                process.wait()
-            except (psutil.NoSuchProcess, psutil.AccessDenied):
-                continue
-    plugins_dir = Path(get_plugins_dir()).resolve()
-    env = os.environ.copy()
-    dependencies = subprocess.run(
-        [
-            str(Path(sys.executable).with_name("abxpkg")),
-            "env",
-            "--install",
-            "--json",
-            f"--lib={config.ABXPKG_LIB_DIR}",
-            f"--deps-from={plugins_dir / 'chrome' / 'config.json'}:required_binaries",
-        ],
-        capture_output=True,
-        text=True,
-        env=env,
-    )
-    if dependencies.returncode:
-        raise click.ClickException(dependencies.stderr.strip())
-    env.update({key: str(value) for key, value in json.loads(dependencies.stdout).items()})
-    env["ARCHIVEBOX_ABX_PLUGINS_DIR"] = str(plugins_dir)
-    env["ACTIVE_PERSONA"] = persona.name
-    env["PERSONAS_DIR"] = str(persona.path.parent)
-    env["NODE_MODULES_DIR"] = str(config.ABXPKG_LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules")
-    payload = config.model_dump(mode="json")
-    payload.update(
-        ACTIVE_PERSONA=persona.name,
-        CHROME_HEADLESS=False,
-        CHROME_USER_DATA_DIR=str(persona.CHROME_USER_DATA_DIR),
-        AUTH_STORAGE_FILE=str(persona.AUTH_STORAGE_FILE or ""),
-        COOKIES_FILE=str(persona.COOKIES_FILE or ""),
-    )
-    script = Path(__file__).parent.parent / "personas" / "open_browser.js"
-    result = subprocess.run(
-        [env.get("NODE_BINARY") or "node", str(script)],
-        input=json.dumps(payload),
-        text=True,
-        env=env,
-    )
-    sys.exit(result.returncode)
-
-
 @main.command("update")
 @click.option("--name", "-n", help="Set new name")
 def update_cmd(name: str | None):

archivebox/cli/archivebox_pluginmap.py

@@ -18,7 +18,7 @@ EVENT_FLOW_DIAGRAM = """
 │         └─ BinaryRequestEvent                                               │
 │              └─ abxpkg BinaryService builtin providers                      │
 │                   └─ BinaryEvent                                            │
-│                        └─ ArchiveBox DB history projection                  │
+│                        └─ BinaryCacheService / project cache backend        │
 │                                                                             │
 │  CrawlEvent                                                                 │
 │    └─ CrawlSetupEvent                                                       │
@@ -48,7 +48,7 @@ def pluginmap(
     Show the current abx-dl event phases and their associated plugin hooks.
 
     This command reflects the new bus-driven runtime, not the legacy ArchiveBox
-    event runtime. Event names are normalized to hook prefixes by
+    state-machine executor. Event names are normalized to hook prefixes by
     stripping a trailing `Event`, then ArchiveBox checks whether any matching
     `on_{EventFamily}__*` scripts actually exist.
     """

archivebox/cli/archivebox_remove.py

@@ -9,6 +9,7 @@ from collections.abc import Iterable
 
 import rich_click as click
 
+from django.db import OperationalError
 from django.db.models import QuerySet
 
 from archivebox.config import CONSTANTS
@@ -33,7 +34,6 @@ def remove(
     after: float | None = None,
     before: float | None = None,
     yes: bool = False,
-    delete: bool = False,
     out_dir: Path = CONSTANTS.DATA_DIR,
     timeout: float | None = None,
     **kwargs,
@@ -87,9 +87,16 @@ def remove(
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
     # an in-flight daemon transaction can interleave instead of deadlocking.
-    # Snapshot.delete() removes and verifies filesystem outputs first, outside
-    # the short database delete transaction. Failed cleanup retains the row.
+    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
+    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
+    # rmtree doesn't hold the lock either.
     #
+    # The SQLite retry wrapper in core/sqlite_backend/base.py re-raises lock
+    # errors when called inside an atomic block (because it can't safely
+    # release+reacquire a transaction), so we wrap each row's delete in our
+    # own retry loop at this outer (non-atomic) level. Each attempt is a
+    # fresh atomic; an exception cleanly rolls it back before we sleep.
+    retry_interval = 1.0
     deleted_snapshot_pks = []
     timed_out = False
     timeout_error = ""
@@ -98,9 +105,23 @@ def remove(
             timed_out = True
             timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
             break
-        deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
-        if deleted_count:
-            deleted_snapshot_pks.append(pk)
+        while True:
+            try:
+                deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
+                if deleted_count:
+                    deleted_snapshot_pks.append(pk)
+                break
+            except OperationalError as err:
+                if "database is locked" not in str(err):
+                    raise
+                remaining_time = deadline - time.monotonic() if deadline is not None else None
+                if remaining_time is not None and remaining_time <= 0:
+                    timed_out = True
+                    timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock."
+                    break
+                time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval)
+        if timed_out:
+            break
 
     all_snapshots = Snapshot.objects.all()
     remaining_count = all_snapshots.count()
@@ -121,7 +142,6 @@ def remove(
 
 @click.command()
 @click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--delete", is_flag=True, help="Compatibility no-op; archived content is deleted automatically")
 @click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
 @snapshot_filter_options(default_filter_type="exact")
 @docstring(remove.__doc__)

archivebox/cli/archivebox_run.py

@@ -54,9 +54,7 @@ RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
 
 
 def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
-    # A supervised daemon must exit non-zero so supervisord restarts it, while
-    # still unwinding the active crawl's hooks and persisted process state.
-    raise SystemExit(128 + sig.value)
+    os._exit(128 + int(sig))
 
 
 def process_stdin_records() -> int:
@@ -69,8 +67,8 @@ def process_stdin_records() -> int:
 
     Outputs JSONL of all processed records (for chaining).
 
-    Handles Crawl and Snapshot work records. ArchiveResult records are accepted
-    as references to their parent Snapshot and plugin.
+    Handles any record type: Crawl, Snapshot, ArchiveResult.
+    Auto-cascades: Crawl → Snapshots → ArchiveResults.
 
     Returns exit code (0 = success, 1 = error).
     """
@@ -87,7 +85,6 @@ def process_stdin_records() -> int:
     )
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot, ArchiveResult
-    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.crawls.models import Crawl
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Binary
@@ -154,15 +151,33 @@ def process_stdin_records() -> int:
                     queued_count += 1
 
             elif record_type == TYPE_ARCHIVERESULT:
-                snapshot_id = str(record.get("snapshot_id") or "")
-                plugin_name = str(record.get("plugin") or "")
-                archiveresult = None
-                if not snapshot_id and record_id:
-                    archiveresult = ArchiveResult.objects.filter(_uuid_ref_query("id", str(record_id))).select_related("snapshot").first()
-                    if archiveresult:
-                        snapshot_id = str(archiveresult.snapshot_id)
-                        plugin_name = plugin_name or archiveresult.plugin
-                snapshot = Snapshot.objects.filter(id=snapshot_id).first() if snapshot_id else None
+                if record_id:
+                    # Existing archiveresult - re-queue
+                    try:
+                        archiveresult = ArchiveResult.objects.get(id=record_id)
+                    except ArchiveResult.DoesNotExist:
+                        archiveresult = None
+                else:
+                    archiveresult = None
+
+                snapshot_id = record.get("snapshot_id")
+                plugin_name = record.get("plugin")
+                snapshot = None
+                if archiveresult:
+                    if archiveresult.status in [
+                        ArchiveResult.StatusChoices.FAILED,
+                        ArchiveResult.StatusChoices.SKIPPED,
+                        ArchiveResult.StatusChoices.NORESULTS,
+                        ArchiveResult.StatusChoices.BACKOFF,
+                    ]:
+                        archiveresult.reset_for_retry()
+                    snapshot = archiveresult.snapshot
+                    plugin_name = plugin_name or archiveresult.plugin
+                elif snapshot_id:
+                    try:
+                        snapshot = Snapshot.objects.get(id=snapshot_id)
+                    except Snapshot.DoesNotExist:
+                        snapshot = None
 
                 if snapshot:
                     snapshot.queue_for_extraction()
@@ -170,7 +185,7 @@ def process_stdin_records() -> int:
                     snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                     if plugin_name:
                         plugin_names_by_crawl[crawl_id].add(str(plugin_name))
-                    output_records.append(archiveresult.to_json() if archiveresult else record)
+                    output_records.append(record if not archiveresult else archiveresult.to_json())
                     queued_count += 1
 
             elif record_type in {TYPE_BINARYREQUEST, TYPE_BINARY}:
@@ -228,6 +243,7 @@ def process_stdin_records() -> int:
                     crawl_id,
                     snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
                     selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
+                    selected_plugins_are_explicit=False,
                 )
     return 0
 
@@ -236,6 +252,7 @@ def run_runner(
     daemon: bool = False,
     crawl_id: str | None = None,
     maintenance_only: bool = False,
+    maintenance_batch_size: int | None = None,
 ) -> int:
     """
     Run the background runner loop.
@@ -270,38 +287,36 @@ def run_runner(
         current.mark_exited()
         return 0
 
-    recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
+    recover_orchestrator_state(include_chrome=True)
     if crawl_id:
         from django.utils import timezone
         from archivebox.crawls.models import Crawl
 
         crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
         now = timezone.now()
-        # Winning the single-runner gate terminates and waits for every older
-        # runner. Requeue the explicitly requested crawl so an abandoned
-        # future ownership lease cannot hide it from the unified scheduler.
-        if crawl is not None:
-            crawl.update_and_requeue(retry_at=now, refresh=False)
-    # Interactivity follows terminal ownership, not the spelling of the command.
-    # A supervised add worker has pipes and receives terminal intent/answers from
-    # its parent; direct run owns the terminal itself. Daemons and redirected
-    # commands must never wait for a person who cannot answer their prompt.
-    # Trust the explicit terminal-owner flag, not Process.root: when add borrows
-    # a server-owned supervisord, its worker's OS ancestry still leads to server.
-    # Gating on root=ADD silently made the first Ctrl+C abort after takeover.
-    interactive_interrupts = not daemon and (
-        (sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())) or os.environ.get("ARCHIVEBOX_INTERACTIVE_INTERRUPTS") == "1"
-    )
+        # Only re-lease when the row is unscheduled (retry_at IS NULL) or its
+        # existing lease has already expired. A future retry_at means another
+        # worker is already scheduled — don't clobber.
+        if crawl is not None and (crawl.retry_at is None or crawl.retry_at <= now):
+            # extra_filter pins the read-time retry_at so a concurrent worker
+            # that grabbed the lease between our SELECT and UPDATE wins.
+            crawl.safe_update(
+                {"retry_at": now},
+                refresh=False,
+                extra_filter={"retry_at": crawl.retry_at},
+            )
+    # Only a foreground `archivebox add` gets the interactive "abort current
+    # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
+    # orchestrators should shut down immediately and cleanly on the first signal.
+    interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
     if daemon:
         os.environ[RUNNER_DAEMON_ENV] = "1"
-
-    shutdown_state = None
     try:
         with (
             foreground_shutdown_signals(
                 on_signal=_exit_daemon_runner_on_signal if daemon else None,
                 raise_on_first_signal=not daemon,
-            ) as shutdown_state,
+            ),
             foreground_parent_watchdog(enabled=not daemon),
         ):
             run_pending_crawls(
@@ -309,17 +324,12 @@ def run_runner(
                 crawl_id=crawl_id,
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
+                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
             )
-        if daemon and shutdown_state.signal_name:
-            return 128 + signal.Signals[shutdown_state.signal_name].value
         return 0
     except KeyboardInterrupt:
-        if daemon and shutdown_state and shutdown_state.signal_name:
-            return 128 + signal.Signals[shutdown_state.signal_name].value
-        return 130
+        return 0
     except asyncio.CancelledError as e:
-        if daemon and shutdown_state and shutdown_state.signal_name:
-            return 128 + signal.Signals[shutdown_state.signal_name].value
         if daemon:
             rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
             return 1
@@ -338,7 +348,14 @@ def run_runner(
 @click.option("--crawl-id", help="Run the crawl runner for a specific crawl only")
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
-@click.option("--maintenance-only", is_flag=True, help="Only process sealed Snapshot maintenance and search-index backfills")
+@click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
+@click.option(
+    "--maintenance-batch-size",
+    type=int,
+    default=None,
+    hidden=True,
+    help="Limit queued maintenance snapshots claimed per scheduler tick",
+)
 @click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
 def main(
     daemon: bool,
@@ -346,6 +363,7 @@ def main(
     snapshot_id: str,
     binary_id: str,
     maintenance_only: bool,
+    maintenance_batch_size: int | None,
     no_stdin: bool,
 ):
     """
@@ -370,13 +388,13 @@ def main(
                 ),
                 foreground_parent_watchdog(enabled=False),
             ):
-                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
         except KeyboardInterrupt:
-            sys.exit(130)
+            sys.exit(0)
 
     with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
         if snapshot_id:
-            sys.exit(run_snapshot_worker(snapshot_id, daemon=daemon))
+            sys.exit(run_snapshot_worker(snapshot_id))
 
         if binary_id:
             try:
@@ -396,22 +414,23 @@ def main(
         if crawl_id:
             sys.exit(
                 run_runner(
-                    daemon=daemon,
+                    daemon=False,
                     crawl_id=crawl_id,
                     maintenance_only=maintenance_only,
+                    maintenance_batch_size=maintenance_batch_size,
                 ),
             )
 
         if maintenance_only:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=True))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))
 
         if not no_stdin and not sys.stdin.isatty():
             sys.exit(process_stdin_records())
         else:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
 
 
-def run_snapshot_worker(snapshot_id: str, *, daemon: bool = False) -> int:
+def run_snapshot_worker(snapshot_id: str) -> int:
     from archivebox.config import CONSTANTS
     from archivebox.core.takeover_util import enter_single_runner_gate
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
@@ -426,7 +445,6 @@ def run_snapshot_worker(snapshot_id: str, *, daemon: bool = False) -> int:
         return 0
 
     snapshot = None
-    interactive_interrupts = not daemon and sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             for _ in range(10):
@@ -435,7 +453,7 @@ def run_snapshot_worker(snapshot_id: str, *, daemon: bool = False) -> int:
                     snapshot.update_and_requeue(retry_at=timezone.now())
                 elif snapshot.retry_at > timezone.now():
                     break
-                if not run_due_snapshot(snapshot, lock_seconds=60, interactive_interrupts=interactive_interrupts):
+                if not run_due_snapshot(snapshot, lock_seconds=60):
                     break
         return 0
     except KeyboardInterrupt:
@@ -448,7 +466,7 @@ def run_snapshot_worker(snapshot_id: str, *, daemon: bool = False) -> int:
                 snapshot.update_and_requeue(retry_at=timezone.now())
         except Exception:
             pass
-        return 130
+        return 0
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
         import traceback

archivebox/cli/archivebox_schedule.py

@@ -37,7 +37,6 @@ def schedule(
         "created_schedule_ids": [],
         "disabled_count": 0,
         "run_all_enqueued": 0,
-        "run_all_maintained": 0,
         "active_schedule_ids": [],
     }
 
@@ -58,12 +57,12 @@ def schedule(
 
         created_by_id = get_or_create_system_user_pk()
         is_update_schedule = not import_path
-        template_urls = import_path or ""
+        template_urls = import_path or "archivebox://update"
         template_label = (f"Scheduled import: {template_urls}" if import_path else "Scheduled ArchiveBox update")[:64]
         template_notes = (
             f"Created by archivebox schedule for {template_urls}"
             if import_path
-            else "Created by archivebox schedule to run recurring ArchiveBox maintenance."
+            else "Created by archivebox schedule to queue recurring archivebox://update maintenance crawls."
         )
 
         template = Crawl.objects.create(
@@ -77,6 +76,7 @@ def schedule(
             retry_at=None,
             config={
                 "DEPTH": 0 if is_update_schedule else depth,
+                "SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
                 # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the
                 # template defaults. Anything left unset falls through to the
                 # standard config stack at crawl-resolution time.
@@ -86,7 +86,6 @@ def schedule(
         crawl_schedule = CrawlSchedule.objects.create(
             template=template,
             schedule=schedule_str,
-            config={**template.config, "SCHEDULE_KIND": "update" if is_update_schedule else "crawl"},
             is_enabled=True,
             label=template_label,
             notes=template_notes,
@@ -120,19 +119,12 @@ def schedule(
 
     if run_all:
         enqueued = 0
-        maintained = 0
         now = timezone.now()
         for scheduled_crawl in schedules:
-            if scheduled_crawl.dispatch(queued_at=now) is None:
-                maintained += 1
-            else:
-                enqueued += 1
+            scheduled_crawl.enqueue(queued_at=now)
+            enqueued += 1
         result["run_all_enqueued"] = enqueued
-        result["run_all_maintained"] = maintained
         print(f"[green]\\[*] Enqueued {enqueued} scheduled crawl(s) immediately.[/green]")
-        if maintained:
-            run_pending_crawls(maintenance_only=True)
-            print(f"[green]\\[*] Ran {maintained} scheduled maintenance update(s).[/green]")
         if enqueued:
             print(
                 "[yellow]\\[*] Start `archivebox server`, `archivebox run --daemon`, or `archivebox schedule --foreground` to process the queued crawls.[/yellow]",
@@ -179,7 +171,7 @@ def schedule(
 @click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
 @click.option("--show", is_flag=True, help="Print all currently enabled schedules")
 @click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
-@click.option("--run-all", is_flag=True, help="Dispatch all enabled schedules immediately")
+@click.option("--run-all", is_flag=True, help="Enqueue all enabled schedules immediately and process them once")
 @click.argument("import_path", required=False)
 @docstring(schedule.__doc__)
 def main(**kwargs):

archivebox/cli/archivebox_server.py

@@ -11,8 +11,6 @@ from collections.abc import Iterable
 
 import rich_click as click
 from rich import print
-from rich.style import Style
-from rich.text import Text
 
 from archivebox.config import CONSTANTS
 from archivebox.misc.util import docstring, enforce_types
@@ -46,13 +44,13 @@ def _split_bind_spec(spec: str) -> tuple[str, str]:
     """Split a ``host:port`` / ``host`` / ``port`` spec into ``(host, port)``.
 
     The empty strings stand in for "not provided"; the caller fills in
-    defaults. Bracketed IPv6 literals like ``[::1]:5797`` are handled.
+    defaults. Bracketed IPv6 literals like ``[::1]:8000`` are handled.
     """
     spec = (spec or "").strip()
     if not spec:
         return "", ""
     if spec.startswith("["):
-        # Bracketed IPv6: ``[::1]`` or ``[::1]:5797``
+        # Bracketed IPv6: ``[::1]`` or ``[::1]:8000``
         end = spec.find("]")
         if end == -1:
             return spec, ""  # malformed; let validator reject it
@@ -77,11 +75,11 @@ def _parse_and_validate_bind_spec(spec: str) -> tuple[str, str]:
     (normalized to ``127.0.0.1``). Bare hostnames are rejected because the
     bind address feeds Daphne, which has to listen on a numeric address;
     public hostnames belong in ``BASE_URL`` instead. Empty values fall back
-    to ``127.0.0.1`` / ``5797``.
+    to ``127.0.0.1`` / ``8000``.
     """
     raw_host, raw_port = _split_bind_spec(spec)
     host = raw_host.strip()
-    port = (raw_port or "").strip() or "5797"
+    port = (raw_port or "").strip() or "8000"
 
     if host == "" or host.lower() == "localhost":
         host = "127.0.0.1"
@@ -198,7 +196,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
             "[yellow]    routing cannot work against an IP address. Set BASE_URL explicitly, e.g.[/yellow]",
         )
         print(
-            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:5797[/yellow]",
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
         )
         if config.USES_SUBDOMAIN_ROUTING:
             print(
@@ -221,7 +219,7 @@ def _print_server_startup_warnings(config, host: str, port: str) -> None:
             "[yellow]    a reverse proxy / ingress / public hostname, e.g.[/yellow]",
         )
         print(
-            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:5797[/yellow]",
+            "[yellow]      BASE_URL=https://archive.example.com archivebox server 0.0.0.0:8000[/yellow]",
         )
         print()
 
@@ -233,7 +231,6 @@ def server(
     debug: bool = False,
     daemonize: bool = False,
     nothreading: bool = False,
-    createsuperuser: bool = False,
 ) -> None:
     """Run the ArchiveBox HTTP server"""
     from archivebox.config.common import get_config
@@ -247,44 +244,27 @@ def server(
 
     from django.contrib.auth.models import User
 
-    if createsuperuser:
-        from archivebox.cli.archivebox_manage import manage
-
-        manage(args=["createsuperuser"])
+    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
+        print()
+        print(
+            "[violet]Hint:[/violet] To create an [bold]admin username & password[/bold] for the [deep_sky_blue3][underline][link=http://{host}:{port}/admin]Admin UI[/link][/underline][/deep_sky_blue3], run:",
+        )
+        print("      [green]archivebox manage createsuperuser[/green]")
         print()
 
     # First non-empty positional arg is the bind spec; otherwise inherit from
-    # config (which defaults to "127.0.0.1:5797"). _parse_and_validate_bind_spec
+    # config (which defaults to "127.0.0.1:8000"). _parse_and_validate_bind_spec
     # hard-errors on hostnames so the rest of the server can assume a numeric
     # bind host.
     bind_spec = next((arg for arg in runserver_args if arg), "")
     host, port = _parse_and_validate_bind_spec(bind_spec)
 
-    from archivebox.personas.models import Persona
-
-    # Older collections deferred this until /add/. Reconcile before serving any
-    # requests, and report ambiguous profile names before Chrome can use them.
-    Persona.get_or_create_default().ensure_dirs()
-
-    if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
-        from archivebox.core.routes_util import build_admin_url
-
-        print()
-        print("[violet]Hint:[/violet] Open the Admin UI to create the first admin and finish web setup:")
-        runtime_config = config.model_copy(update={"BIND_ADDR": f"{host}:{port}"})
-        print(f"      [green]{build_admin_url('/admin/', config=runtime_config)}[/green]")
-        if not config.BASE_URL and host not in ("127.0.0.1", "localhost"):
-            print("      (When running remotely, replace localhost with this server's IP address or hostname.)")
-        print()
-
     if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
-        from archivebox.workers.supervisord_util import resolve_env_binary
-
         log_path = CONSTANTS.LOGS_DIR / "server.log"
         log_path.parent.mkdir(parents=True, exist_ok=True)
         daemon_env = os.environ.copy()
         daemon_env["ARCHIVEBOX_SERVER_DAEMON_CHILD"] = "1"
-        daemon_cmd = [str(resolve_env_binary("archivebox")), "server"]
+        daemon_cmd = [sys.executable, "-m", "archivebox", "server"]
         if debug:
             daemon_cmd.append("--debug")
         if reload:
@@ -313,14 +293,7 @@ def server(
             except OSError:
                 time.sleep(0.1)
         else:
-            print(f"[red][X] ArchiveBox daemon server pid={proc.pid} did not become ready. See {log_path}[/red]")
-            proc.terminate()
-            try:
-                proc.wait(timeout=10)
-            except subprocess.TimeoutExpired:
-                proc.kill()
-                proc.wait(timeout=5)
-            sys.exit(1)
+            print(f"[yellow][!] ArchiveBox daemon server pid={proc.pid} is still starting. See {log_path}[/yellow]")
         return
 
     os.environ["BIND_ADDR"] = f"{host}:{port}"
@@ -350,15 +323,12 @@ def server(
         print("[green][+] Starting ArchiveBox webserver in DEBUG mode...[/green]")
     else:
         print("[green][+] Starting ArchiveBox webserver...[/green]")
-    bind_url = f"http://{host}:{port}"
-    bind_message = Text.from_markup(
-        "    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [dim]BIND_ADDR[/dim] ",
+    print(
+        f"    [blink][green]>[/green][/blink] Starting ArchiveBox webserver on [dim]BIND_ADDR[/dim] [deep_sky_blue4][link=http://{host}:{port}]http://{host}:{port}[/link][/deep_sky_blue4]",
+    )
+    print(
+        f"    [green]>[/green] Log in to ArchiveBox Admin UI on [dim]BASE_URL [/dim] [deep_sky_blue3][link={admin_url}]{admin_url}[/link][/deep_sky_blue3]",
     )
-    bind_message.append(bind_url, style=Style(color="deep_sky_blue4", link=bind_url))
-    print(bind_message)
-    admin_message = Text.from_markup("    [green]>[/green] Log in to ArchiveBox Admin UI on [dim]BASE_URL [/dim] ")
-    admin_message.append(admin_url, style=Style(color="deep_sky_blue3", link=admin_url))
-    print(admin_message)
     print("    > Writing ArchiveBox error log to ./logs/errors.log")
     print()
 
@@ -366,13 +336,16 @@ def server(
     # security-mode + base-url warnings see the effective values.
     runtime_config = get_config()
     _print_server_startup_warnings(runtime_config, host, port)
+    bind_url = f"http://{host}:{port}"
     command = current_command(Process.TypeChoices.SERVER, data_dir=CONSTANTS.DATA_DIR, url=bind_url)
 
     def still_owns_runtime_stack() -> bool:
-        # This is polled at the log-tail cadence. Keep the autocommit
-        # connection open so SQLite can reuse its page cache between polls;
-        # reconnecting here repeatedly rereads the process table while idle.
-        return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
+        from django.db import connections
+
+        try:
+            return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
+        finally:
+            connections.close_all()
 
     shutdown_state = None
     try:
@@ -386,7 +359,7 @@ def server(
                     data_dir=CONSTANTS.DATA_DIR,
                     exclude_id=command.id,
                 )
-                takeover_components = active_supervisord_runtime_components()
+                takeover_components = active_supervisord_runtime_components(config=config)
                 if older_owner and takeover_components:
                     print(
                         "[yellow][*] Taking over "
@@ -420,13 +393,8 @@ def server(
     except KeyboardInterrupt:
         pass
     finally:
-        from django.db import connections
-
-        try:
-            if not shutdown_state or not shutdown_state.signal_name:
-                command.mark_exited()
-        finally:
-            connections.close_all()
+        if not shutdown_state or not shutdown_state.signal_name:
+            command.mark_exited()
     print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
 
 
@@ -436,7 +404,6 @@ def server(
 @click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
 @click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
 @click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
-@click.option("--createsuperuser", is_flag=True, help="Run archivebox manage createsuperuser before starting the server")
 @docstring(server.__doc__)
 def main(**kwargs):
     server(**kwargs)

archivebox/cli/archivebox_snapshot.py

@@ -31,69 +31,14 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox snapshot"
 
 import sys
-from collections.abc import Iterable, Iterator
-from itertools import islice
+from collections.abc import Iterable
 
 import rich_click as click
 from rich import print as rprint
 from django.db.models import QuerySet
 
 SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
-SNAPSHOT_LIST_CHUNK_SIZE = 5000
-
-
-def iter_snapshot_json(queryset: QuerySet) -> Iterator[dict[str, object]]:
-    from archivebox.config import VERSION
-    from archivebox.config.common import redact_sensitive_config
-    from archivebox.core.models import SnapshotTag
-
-    fields = (
-        "id",
-        "crawl_id",
-        "url",
-        "title",
-        "bookmarked_at",
-        "created_at",
-        "modified_at",
-        "config",
-        "notes",
-        "timestamp",
-        "depth",
-        "status",
-        "fs_version",
-        "output_size",
-    )
-    rows = queryset.values(*fields).iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE)
-    while batch := list(islice(rows, SNAPSHOT_LIST_CHUNK_SIZE)):
-        tags_by_snapshot = {row["id"]: [] for row in batch}
-        tag_rows = (
-            SnapshotTag.objects.filter(snapshot_id__in=tags_by_snapshot).order_by("tag__name").values_list("snapshot_id", "tag__name")
-        )
-        for snapshot_id, tag_name in tag_rows:
-            tags_by_snapshot[snapshot_id].append(tag_name)
-
-        for row in batch:
-            archive_size = int(row["output_size"] or 0)
-            yield {
-                "type": "Snapshot",
-                "schema_version": VERSION,
-                "id": str(row["id"]),
-                "crawl_id": str(row["crawl_id"]),
-                "url": row["url"],
-                "title": row["title"],
-                "tags": ",".join(sorted(tags_by_snapshot[row["id"]])),
-                "bookmarked_at": row["bookmarked_at"].isoformat() if row["bookmarked_at"] else None,
-                "created_at": row["created_at"].isoformat() if row["created_at"] else None,
-                "modified_at": row["modified_at"].isoformat() if row["modified_at"] else None,
-                "config": redact_sensitive_config(row["config"]),
-                "notes": row["notes"],
-                "timestamp": row["timestamp"],
-                "depth": row["depth"],
-                "status": row["status"],
-                "fs_version": row["fs_version"],
-                "archive_size": archive_size,
-                "output_size": archive_size,
-            }
+SNAPSHOT_LIST_CHUNK_SIZE = 100
 
 
 # =============================================================================
@@ -298,7 +243,7 @@ def list_snapshots(
     Exit codes:
         0: Success (even if no results)
     """
-    from archivebox.misc.jsonl import write_records
+    from archivebox.misc.jsonl import write_record
 
     output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
     if output_formats > 1:
@@ -351,7 +296,9 @@ def list_snapshots(
         return 0
 
     if not is_tty:
-        count = write_records(iter_snapshot_json(queryset))
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+            write_record(snapshot.to_json())
+            count += 1
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0
 

archivebox/cli/archivebox_status.py

@@ -44,18 +44,12 @@ def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
         legacy_snapshot_dirs = [
             entry for entry in archive_dir.iterdir() if entry.is_dir() and not entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry)
         ]
-    from archivebox.misc.db import database_display_location, is_postgres
-
-    index_location = database_display_location() if is_postgres() else CONSTANTS.SQL_INDEX_FILENAME
-    print(f"    > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {index_location})")
+    print(f"    > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {CONSTANTS.SQL_INDEX_FILENAME})")
     print(f"    > JSON Link Details: {len(legacy_snapshot_dirs)} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
     print()
     print("[green]\\[*] Scanning archive data directories...[/green]")
     users_dir = CONSTANTS.USERS_DIR
-    # The current users layout is nested below archive/, so scanning both
-    # archive/ and archive/users/ recursively counts every current snapshot
-    # twice. A single archive root includes both legacy and current layouts.
-    scan_roots = [archive_dir] if archive_dir.exists() else []
+    scan_roots = [root for root in (archive_dir, users_dir) if root.exists()]
     scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir)
     print(f"[yellow]   {scan_roots_display}[/yellow]")
     do_precise_fs_scan = num_sql_links <= MAX_STATUS_FS_DIR_SCAN
@@ -115,7 +109,7 @@ def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
 
     if orphaned_dirs:
         print("    [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")
-        print("        [green]archivebox update[/green]")
+        print("        [green]archivebox init[/green]")
 
     print()
     print("[green]\\[*] Scanning recent archive changes and user logins:[/green]")
@@ -131,11 +125,9 @@ def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
         print(f"    Last changes: {str(last_downloaded.downloaded_at)[:16]}")
 
     if not users:
-        from archivebox.core.routes_util import build_admin_url
-
         print()
-        print("    [violet]Hint:[/violet] Open the Admin UI to create the first admin and finish web setup:")
-        print(f"        [green]{build_admin_url('/admin/', config=config)}[/green]")
+        print("    [violet]Hint:[/violet] You can create an admin user by running:")
+        print("        [green]archivebox manage createsuperuser[/green]")
 
     print()
     recent_snapshots = snapshots_qs.order_by(

archivebox/cli/archivebox_update.py

@@ -7,7 +7,6 @@ import os
 import asyncio
 import shlex
 import time
-from itertools import islice
 
 from typing import TYPE_CHECKING, Any
 from collections.abc import Iterable
@@ -53,20 +52,17 @@ def _get_search_indexing_plugins() -> list[str]:
 def _build_filtered_snapshots_queryset(
     **kwargs,
 ):
+    from archivebox.core.models import Snapshot
     from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
     limit = kwargs.pop("limit", None)
-    reverse = kwargs.pop("reverse", False)
     snapshots = build_snapshot_queryset(**kwargs)
-    snapshots = snapshots.order_by(kwargs.get("sort") or "-created_at", "-id")
-    if reverse:
-        snapshots = snapshots.reverse()
     if kwargs.get("resume"):
-        snapshots = snapshots.filter(**{"timestamp__gte" if reverse else "timestamp__lte": kwargs["resume"]})
+        snapshots = snapshots.filter(timestamp__lte=kwargs["resume"])
     snapshots = snapshots.select_related("crawl")
     if limit is not None and limit > 0:
         snapshot_ids = list(snapshots.values_list("id", flat=True)[:limit])
-        snapshots = snapshots.filter(id__in=snapshot_ids)
+        snapshots = Snapshot.objects.filter(id__in=snapshot_ids).select_related("crawl")
 
     return snapshots
 
@@ -77,73 +73,104 @@ def reindex_snapshots(
     search_plugins: list[str],
     batch_size: int,
     collect_ids: bool = False,
-    reverse: bool = False,
     wait_for_turn=None,
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
-    from archivebox.core.models import ArchiveResult, Snapshot
-    from django.db.models import Exists, OuterRef
+    from archivebox.core.models import ArchiveResult
+    from abx_dl.models import discover_plugins
 
-    # Search backfill is the one maintenance hook allowed to execute without
-    # reopening a Snapshot. Restrict that exception to already-sealed rows;
-    # every open lifecycle state remains owned by the normal runner lifecycle.
-    snapshots = snapshots.filter(status=Snapshot.StatusChoices.SEALED)
+    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
+    records: list[dict[str, str]] = []
+    plugins_by_name = discover_plugins(runtime="archivebox")
+    required_hooks_by_plugin = {
+        plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
+        for plugin_name in search_plugins
+        if plugin_name in plugins_by_name
+    }
 
-    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
-    print(f"[*] Reindexing missing search indexes with: {', '.join(search_plugins)}")
+    total = snapshots.count()
+    print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
 
-    completed_statuses = [ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS]
-    for plugin_name in search_plugins:
-        completed_result = ArchiveResult.objects.filter(
-            snapshot_id=OuterRef("pk"),
-            plugin=plugin_name,
-            status__in=completed_statuses,
-        )
-        candidates = snapshots.annotate(has_completed_index=Exists(completed_result)).filter(has_completed_index=False)
-        candidates = candidates.order_by("created_at", "id") if reverse else candidates.order_by("-created_at", "-id")
-        remaining = candidates.select_related(None).only("id", "timestamp", "created_at").paged_iterator(chunk_size=batch_size)
-        while True:
-            if wait_for_turn:
-                wait_for_turn()
-            batch = list(islice(remaining, batch_size))
-            if not batch:
-                break
-            records = [{"type": "Snapshot", "id": str(snapshot.id)} for snapshot in batch]
-            stats["processed"] += len(batch)
-            stats["requested"] += len(batch)
-            if collect_ids:
-                stats["snapshot_ids"].extend(str(snapshot.id) for snapshot in batch)
-            exit_code = run_plugins(
-                args=(),
-                records=records,
-                plugins=plugin_name,
-                wait=True,
-                emit_results=False,
-                show_progress=False,
+    def run_batch() -> None:
+        if not records:
+            return
+        if wait_for_turn:
+            wait_for_turn()
+        batch_records = list(records)
+        snapshot_ids = {record["snapshot_id"] for record in batch_records}
+        plugin_names = {record["plugin"] for record in batch_records}
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=snapshot_ids,
+                plugin__in=plugin_names,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        records_to_queue = []
+        for record in batch_records:
+            snapshot_id = record["snapshot_id"]
+            plugin_name = record["plugin"]
+            required_hooks = required_hooks_by_plugin.get(plugin_name, frozenset())
+            if required_hooks and all((snapshot_id, plugin_name, hook_name) in queued_rows for hook_name in required_hooks):
+                stats["skipped_queued"] += 1
+                continue
+            records_to_queue.append(record)
+        if not records_to_queue:
+            print(
+                f"    [{stats['processed']}/{total}] Already queued {len(batch_records)} index jobs",
             )
-            if exit_code != 0:
-                raise SystemExit(exit_code)
-            stats["reindexed"] += len(batch)
-            print(f"    [{plugin_name}] indexed {stats['reindexed']} missing snapshots")
-    return stats
-
-
-def run_scheduled_maintenance(*, batch_size: int = 500) -> dict[str, Any]:
-    """Queue stale filesystem rows and backfill missing search facts."""
-    from archivebox.core.models import Snapshot
-
-    filesystem_stats = process_all_db_snapshots(batch_size=batch_size)
-    search_plugins = _get_search_indexing_plugins()
-    search_stats = (
-        reindex_snapshots(
-            Snapshot.objects.filter(fs_version=Snapshot._fs_current_version()),
-            search_plugins=search_plugins,
-            batch_size=batch_size,
+            records.clear()
+            return
+        # `archivebox update --index-only` intentionally breaks the usual
+        # "runner discovers work" rule by inserting synthetic queued
+        # ArchiveResult rows for search backends. run_plugins() keeps this as
+        # statement-sized UPDATE/bulk_create work, then bumps Snapshot.retry_at
+        # so the orchestrator owns actual hook execution. Paused snapshots stay
+        # PAUSED; run_due_snapshot restores retry_at=MAX after targeted rows
+        # finish.
+        exit_code = run_plugins(
+            args=(),
+            records=records_to_queue,
+            wait=False,
+            emit_results=False,
+            show_progress=False,
+            preserve_queued=True,
         )
-        if search_plugins
-        else {"processed": 0, "requested": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
-    )
-    return {"filesystem": filesystem_stats, "search": search_stats}
+        if exit_code != 0:
+            raise SystemExit(exit_code)
+        stats["queued"] += len(records_to_queue)
+        print(
+            f"    [{stats['processed']}/{total}] Queued {len(records_to_queue)} index jobs for orchestrator",
+        )
+        records.clear()
+
+    for snapshot in snapshots.select_related("crawl").paged_iterator(chunk_size=batch_size):
+        try:
+            stats["processed"] += 1
+
+            if _get_snapshot_crawl(snapshot) is None:
+                continue
+
+            if collect_ids:
+                stats["snapshot_ids"].append(str(snapshot.id))
+            for plugin_name in search_plugins:
+                records.append(
+                    {
+                        "type": "ArchiveResult",
+                        "snapshot_id": str(snapshot.id),
+                        "plugin": plugin_name,
+                    },
+                )
+                stats["requested"] += 1
+            if len(records) >= batch_size:
+                run_batch()
+        except KeyboardInterrupt as err:
+            err.archivebox_resume = snapshot.timestamp
+            raise
+
+    run_batch()
+    return stats
 
 
 @enforce_types
@@ -165,18 +192,18 @@ def update(
     continuous: bool = False,
     index_only: bool = False,
     migrate_only: bool = False,
-    rescan: bool = False,
-    reverse: bool = False,
     stop_daemon_stack: bool = True,
 ) -> None:
     """
-    Update the database without forcing a filesystem scan by default.
+    Update snapshots: migrate old dirs, reconcile DB, and re-queue for archiving.
 
-    --migrate-only: eagerly finish pending filesystem migrations for known rows.
-    --index-only: reconcile known snapshot assets/metadata and backfill search.
-    --rescan: discover filesystem orphans first, then repair known snapshots.
-    Explicit scans default to newest first; --reverse selects oldest first.
-    Filters select existing database snapshots for targeted updates.
+    Three-phase operation (without filters):
+    - Phase 1: Drain old archive/ dirs by moving to new fs location (0.8.x → 0.9.x)
+    - Phase 2: O(n) scan over entire DB from most recent to least recent
+    - No orphan scans needed (trust 1:1 mapping between DB and filesystem after phase 1)
+
+    With filters: Only phase 2 (DB query), no filesystem operations.
+    Without filters: All phases (full update).
     """
 
     from rich import print
@@ -200,6 +227,8 @@ def update(
     from archivebox.core.takeover_util import (
         command_owns_foreground_runner,
         current_command,
+        ensure_daemon_stack,
+        foreground_runner_owner,
         standby_until_foreground_runner_needed,
     )
     from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
@@ -219,9 +248,11 @@ def update(
         standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
         raise_if_shutdown_requested()
 
-    def run_scoped_runner(*args: str) -> None:
+    def run_scoped_runner(*args: str, ensure_daemon_reason: str | None = None) -> None:
         while True:
             wait_for_turn()
+            if ensure_daemon_reason:
+                ensure_daemon_stack(reason=ensure_daemon_reason)
             exit_code = run_runner_worker(
                 list(args),
                 name=f"worker_runner_update_{os.getpid()}",
@@ -252,28 +283,17 @@ def update(
     exit_code = 0
 
     try:
-        if rescan and (is_filtered_update or resume or continuous):
-            raise click.UsageError("--rescan cannot be combined with filters, --resume, or --continuous; rerun it to resume automatically.")
-        if rescan:
-            migrate_only = True  # Discovery must not schedule new captures.
         wait_for_turn()
 
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             while True:
-                if rescan:
-                    from archivebox.cli.rescan import rescan_snapshots
-
-                    rescan_snapshots(
-                        wait_for_turn=wait_for_turn,
-                        reverse=reverse,
-                        scan_legacy=lambda: drain_old_archive_dirs(batch_size=batch_size, reverse=reverse),
-                    )
-                do_migrate = migrate_only or is_filtered_update and not index_only
-                do_index = index_only
-                if not do_migrate and not do_index:
-                    print(
-                        "[*] Collection ready. Filesystem migrations remain lazy; use --migrate-only, --index-only, or --rescan for explicit scans.",
-                    )
+                do_migrate = migrate_only or not index_only
+                do_index = index_only or not migrate_only
+                do_run_until_idle = do_migrate or do_index
+                ran_post_migrate_runner = False
+                full_update_empty = False
+                maintenance_work_queued = False
+                runner_work_queued = False
 
                 if do_migrate:
                     if (
@@ -305,86 +325,182 @@ def update(
                             after=after,
                             resume=resume,
                             batch_size=batch_size,
-                            queue_for_archiving=not migrate_only,
-                            reverse=reverse,
+                            queue_for_archiving=do_run_until_idle,
                             wait_for_turn=wait_for_turn,
                         )
                         print_stats(stats)
                         touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        maintenance_work_queued = stats.get("queued", 0) > 0
+                        runner_work_queued = runner_work_queued or maintenance_work_queued
                     else:
                         stats_combined = {"phase1": {}, "phase2": {}}
 
-                        print("[*] Phase 2: Selecting database snapshots with stale filesystem versions...")
+                        print("[*] Phase 1: Draining old archive/ directories (0.8.x → 0.9.x migration)...")
+                        stats_combined["phase1"] = drain_old_archive_dirs(
+                            resume_from=resume,
+                            batch_size=batch_size,
+                        )
+
+                        print("[*] Phase 2: Processing all database snapshots (most recent first)...")
                         stats_combined["phase2"] = process_all_db_snapshots(
                             batch_size=batch_size,
                             resume=resume,
                             wait_for_turn=wait_for_turn,
-                            reverse=reverse,
-                            migrate=True,
                         )
                         print_combined_stats(stats_combined)
-                    # The due selectors are indexed and cheap when empty, so
-                    # always drain them instead of preceding the runner with
-                    # whole-table counts merely to decide whether to call it.
-                    print("[*] Phase 3: Running filesystem maintenance until idle...")
-                    if is_filtered_update:
-                        for snapshot_id in sorted(touched_snapshot_ids, reverse=not reverse):
-                            run_scoped_runner("--snapshot-id", snapshot_id)
-                    elif migrate_only:
-                        run_scoped_runner("--maintenance-only")
-                    else:
-                        run_scoped_runner()
+                        full_update_empty = (
+                            stats_combined["phase1"].get("processed", 0) == 0 and stats_combined["phase2"].get("snapshots", 0) == 0
+                        )
+                        maintenance_work_queued = any(
+                            (
+                                stats_combined["phase1"].get("queued", 0),
+                                stats_combined["phase2"].get("queued", 0),
+                                stats_combined["phase2"].get("crawls_sealed", 0),
+                            ),
+                        )
+                        runner_work_queued = runner_work_queued or maintenance_work_queued
+
+                    if do_run_until_idle:
+                        # Filesystem migration is maintenance on existing
+                        # Snapshot rows: Snapshot.save() moves archive/<ts> to
+                        # the current output_dir and preserves the lifecycle
+                        # status. Drain those retry_at ticks before queuing
+                        # search backfill below. Otherwise the sealed/paused
+                        # runner branch correctly sees queued ArchiveResult
+                        # rows first, runs the targeted plugins, and may leave
+                        # the fs_version maintenance tick hidden behind that
+                        # plugin work until another update pass.
+                        if full_update_empty:
+                            print("[*] No snapshots or legacy archive directories found; skipping filesystem maintenance runner.")
+                        elif not maintenance_work_queued:
+                            print("[*] No filesystem maintenance work queued; skipping filesystem maintenance runner.")
+                        else:
+                            print("[*] Phase 3: Running filesystem maintenance until idle...")
+                        if full_update_empty:
+                            pass
+                        elif not maintenance_work_queued:
+                            pass
+                        elif is_filtered_update:
+                            if not touched_snapshot_ids:
+                                print("[*] No matching snapshots queued work for the runner.")
+                            for snapshot_id in sorted(touched_snapshot_ids):
+                                run_scoped_runner("--snapshot-id", snapshot_id)
+                        else:
+                            run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
+                        ran_post_migrate_runner = True
 
                 if do_index:
-                    from archivebox.cli.rescan import reconcile_known_snapshots
-
-                    snapshots = _build_filtered_snapshots_queryset(
-                        filter_patterns=filter_patterns,
-                        filter_type=filter_type,
-                        status=status,
-                        url__icontains=url__icontains,
-                        url__istartswith=url__istartswith,
-                        tag=tag,
-                        crawl_id=crawl_id,
-                        limit=limit,
-                        sort=sort,
-                        search=search,
-                        before=before,
-                        after=after,
-                        resume=resume,
-                        reverse=reverse,
-                    )
-                    reconcile_known_snapshots(snapshots, wait_for_turn=wait_for_turn)
-                    search_plugins = _get_search_indexing_plugins()
-                    if not search_plugins:
-                        print("[*] No search indexing plugins are available, nothing to backfill.")
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping search indexing backfill.")
                     else:
-                        snapshots = _build_filtered_snapshots_queryset(
-                            filter_patterns=filter_patterns,
-                            filter_type=filter_type,
-                            status=status,
-                            url__icontains=url__icontains,
-                            url__istartswith=url__istartswith,
-                            tag=tag,
-                            crawl_id=crawl_id,
-                            limit=limit,
-                            sort=sort,
-                            search=search,
-                            before=before,
-                            after=after,
-                            resume=resume,
-                            reverse=reverse,
+                        ensure_daemon_stack(reason="search indexing")
+                        search_plugins = _get_search_indexing_plugins()
+                        if not search_plugins:
+                            print("[*] No search indexing plugins are available, nothing to backfill.")
+                        else:
+                            snapshots = _build_filtered_snapshots_queryset(
+                                filter_patterns=filter_patterns,
+                                filter_type=filter_type,
+                                status=status,
+                                url__icontains=url__icontains,
+                                url__istartswith=url__istartswith,
+                                tag=tag,
+                                crawl_id=crawl_id,
+                                limit=limit,
+                                sort=sort,
+                                search=search,
+                                before=before,
+                                after=after,
+                                resume=resume,
+                            )
+                            from django.db.models import Exists, OuterRef, Q
+                            from django.utils import timezone
+                            from archivebox.core.models import ArchiveResult, Snapshot
+
+                            scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
+                            queued_index_results = ArchiveResult.objects.filter(
+                                status=ArchiveResult.StatusChoices.QUEUED,
+                                plugin__in=search_plugins,
+                            )
+                            if scoped_snapshot_ids is not None:
+                                queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
+
+                            if queued_index_results.exists():
+                                runner_work_queued = True
+                                now = timezone.now()
+                                queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
+                                snapshots_to_wake = (
+                                    Snapshot.objects.filter(
+                                        status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
+                                    )
+                                    .annotate(
+                                        has_queued_index_result=Exists(queued_result_for_snapshot),
+                                    )
+                                    .filter(
+                                        has_queued_index_result=True,
+                                    )
+                                    .filter(
+                                        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+                                    )
+                                )
+                                if scoped_snapshot_ids is not None:
+                                    snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
+                                woken_count = snapshots_to_wake.update(
+                                    retry_at=now,
+                                    modified_at=now,
+                                )
+                                print(
+                                    "[*] Existing queued search index jobs found; "
+                                    f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
+                                )
+                            else:
+                                collect_index_ids = (
+                                    is_filtered_update
+                                    or foreground_runner_owner(
+                                        data_dir=CONSTANTS.DATA_DIR,
+                                        exclude_id=command.id,
+                                    )
+                                    is not None
+                                )
+                                stats = reindex_snapshots(
+                                    snapshots,
+                                    search_plugins=search_plugins,
+                                    batch_size=batch_size,
+                                    collect_ids=collect_index_ids,
+                                    wait_for_turn=wait_for_turn,
+                                )
+                                print_index_stats(stats)
+                                touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                                runner_work_queued = runner_work_queued or stats["queued"] > 0
+
+                if do_run_until_idle and (do_index or not ran_post_migrate_runner):
+                    # Search/index backfill intentionally queues targeted
+                    # ArchiveResult rows without reopening sealed/paused
+                    # snapshots. This second runner pass drains those plugin
+                    # rows after filesystem maintenance has had its own turn.
+                    # For a normal unfiltered `archivebox update`, keep the
+                    # historical final pass broad enough to resume genuinely
+                    # queued/interrupted crawl work after maintenance is done.
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping queued/interrupted crawl runner.")
+                    elif not runner_work_queued:
+                        print("[*] No queued/interrupted crawl work found; skipping queued/interrupted crawl runner.")
+                    else:
+                        print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
+                    if full_update_empty:
+                        pass
+                    elif not runner_work_queued:
+                        pass
+                    elif touched_snapshot_ids and is_filtered_update:
+                        if not touched_snapshot_ids:
+                            print("[*] No matching snapshots queued work for the runner.")
+                        for snapshot_id in sorted(touched_snapshot_ids):
+                            run_scoped_runner("--snapshot-id", snapshot_id)
+                    else:
+                        run_scoped_runner(
+                            *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
+                            ensure_daemon_reason="search indexing" if do_index else None,
                         )
-                        stats = reindex_snapshots(
-                            snapshots,
-                            search_plugins=search_plugins,
-                            batch_size=batch_size,
-                            collect_ids=is_filtered_update,
-                            reverse=reverse,
-                            wait_for_turn=wait_for_turn,
-                        )
-                        print_index_stats(stats)
-                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))
 
                 if not continuous:
                     break
@@ -396,10 +512,6 @@ def update(
         exit_code = 130
         exact_resume = err.__dict__.get("archivebox_resume")
         resume_cmd = ["archivebox", "update"]
-        if rescan:
-            resume_cmd.append("--rescan")
-        if reverse:
-            resume_cmd.append("--reverse")
         if migrate_only:
             resume_cmd.append("--migrate-only")
         if index_only:
@@ -447,24 +559,25 @@ def update(
             stop_own_supervisord_process()
 
 
-def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500, reverse: bool = False) -> dict[str, int]:
+def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500) -> dict[str, int]:
     """
     Drain old archive/ directories (0.8.x → 0.9.x migration).
 
-    Removes obsolete timestamp symlinks and processes real legacy directories.
+    Only processes real directories (skips symlinks - those are already migrated).
     For each old dir found in archive/:
       1. Load or create DB snapshot
       2. Trigger fs migration on save() to move to data/archive/users/{user}/...
-      3. Remove the old timestamp path after the verified migration commits
+      3. Leave symlink in archive/ pointing to new location
 
-    After this drains, current snapshot data exists only under archive/users/.
+    After this drains, archive/ should only contain symlinks and we can trust
+    1:1 mapping between DB and filesystem.
     """
     from archivebox.core.models import Snapshot
     from archivebox.config import CONSTANTS
     from archivebox.crawls.models import Crawl
     from django.utils import timezone
 
-    stats = {"processed": 0, "migrated": 0, "queued": 0, "skipped": 0, "invalid": 0, "removed_symlinks": 0}
+    stats = {"processed": 0, "migrated": 0, "queued": 0, "skipped": 0, "invalid": 0}
     crawl_url_lines: dict[str, list[str]] = {}
     crawl_url_sets: dict[str, set[str]] = {}
     dirty_crawl_ids: set[str] = set()
@@ -495,36 +608,21 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
             if changed:
                 Crawl.objects.filter(pk=crawl.pk).update(urls="\n".join(lines), modified_at=timezone.now())
 
-    # Compatibility timestamp projections are harmful in portable exports and
-    # duplicate the obsolete 0.7.x-looking namespace without containing data.
+    # Scan for real directories only (skip symlinks - they're already migrated)
     all_entries = list(os.scandir(archive_dir))
-    for entry in all_entries:
-        entry_path = Path(entry.path)
-        if entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry_path):
-            try:
-                target_path = entry_path.resolve(strict=True)
-                points_into_current_layout = target_path.is_relative_to(CONSTANTS.USERS_DIR.resolve(strict=True))
-            except OSError:
-                points_into_current_layout = False
-
-            if points_into_current_layout:
-                entry_path.unlink(missing_ok=True)
-                stats["removed_symlinks"] += 1
-
-    # Scan real legacy directories only; these still contain data to migrate.
     entries = [
-        (float(e.name), e.path)
+        (e.stat().st_mtime, e.path)
         for e in all_entries
         if e.is_dir(follow_symlinks=False) and Snapshot.is_legacy_archive_dir(Path(e.path))  # Skip symlinks and 0.9.x roots
     ]
-    entries.sort(reverse=not reverse)  # Newest first
+    entries.sort(reverse=True)  # Newest first
     print(f"[*] Found {len(entries)} old directories to drain")
 
     for mtime, entry_path in entries:
         entry_path = Path(entry_path)
 
         # Resume from timestamp if specified
-        if resume_from and (entry_path.name < resume_from if reverse else entry_path.name > resume_from):
+        if resume_from and entry_path.name > resume_from:
             continue
 
         stats["processed"] += 1
@@ -626,15 +724,19 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
     return stats
 
 
-def process_all_db_snapshots(
-    batch_size: int = 500,
-    resume: str | None = None,
-    wait_for_turn=None,
-    reverse: bool = False,
-    migrate: bool = False,
-) -> dict[str, int]:
-    """Queue only snapshots whose indexed filesystem version is stale."""
+def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, wait_for_turn=None) -> dict[str, int]:
+    """
+    O(n) scan over entire DB from most recent to least recent.
+
+    For each snapshot:
+      1. Reconcile index.json with DB (merge titles, tags, archive results)
+      2. Mark migrated snapshots sealed unless explicitly re-queued elsewhere
+
+    No orphan detection needed - we trust 1:1 mapping between DB and filesystem
+    after Phase 1 has drained all old archive/ directories.
+    """
     from archivebox.core.models import Snapshot
+    from archivebox.crawls.models import Crawl
     from django.db.models import Q
     from django.utils import timezone
 
@@ -647,41 +749,128 @@ def process_all_db_snapshots(
         "sealed": 0,
         "crawls_sealed": 0,
     }
-    queryset = Snapshot.objects.filter(fs_version__in=Snapshot._FS_VERSION_MIGRATION_PATHS)
+    current_fs_version = Snapshot._fs_current_version()
+
+    queryset = Snapshot.objects.all()
     if resume:
-        queryset = queryset.filter(**{"timestamp__gte" if reverse else "timestamp__lte": resume})
-    initial_now = timezone.now()
-    rows_to_wake = queryset if migrate else queryset.filter(Q(retry_at__isnull=True) | Q(retry_at__gt=initial_now))
-    ordered = rows_to_wake.order_by("created_at", "id") if reverse else rows_to_wake.order_by("-created_at", "-id")
-    remaining = ordered.select_related("crawl__created_by").paged_iterator(chunk_size=batch_size)
-    while True:
+        queryset = queryset.filter(timestamp__lte=resume)
+    total = queryset.count()
+    stats["snapshots"] = total
+    print(f"[*] Processing {total} snapshots from database (most recent first)...")
+
+    def update_in_batches(rows, *, label: str, **updates) -> int:
+        updated = 0
+        checked = 0
+        while True:
+            if wait_for_turn:
+                wait_for_turn()
+            batch = list(rows.only("id", "modified_at").order_by("-timestamp")[:batch_size])
+            if not batch:
+                if updated:
+                    print(f"    [{label}] complete: {updated} rows updated")
+                return updated
+            checked += len(batch)
+            print(f"    [{label}] updating next {len(batch)} rows (seen {checked})...")
+            for snapshot in batch:
+                # This maintenance scan intentionally bypasses save(); it is
+                # only normalizing scheduler fields, and Snapshot.save() may
+                # do filesystem migration work that belongs in the runner.
+                # Guard each single-row UPDATE with modified_at so stale scan
+                # pages cannot overwrite newer runner/admin writes.
+                updated += int(
+                    snapshot.safe_update(
+                        updates,
+                        refresh=False,
+                        extra_filter={"modified_at": snapshot.modified_at},
+                    ),
+                )
+            print(f"    [{label}] updated {updated} rows so far")
+
+    now = timezone.now()
+    updated_rows = update_in_batches(
+        queryset.exclude(
+            status__in=[
+                Snapshot.StatusChoices.QUEUED,
+                Snapshot.StatusChoices.STARTED,
+                Snapshot.StatusChoices.PAUSED,
+                Snapshot.StatusChoices.SEALED,
+            ],
+        ),
+        label="snapshot status normalization",
+        status=Snapshot.StatusChoices.SEALED,
+        retry_at=None,
+        modified_at=now,
+    )
+    stats["sealed"] += updated_rows
+    stats["updated_db"] += updated_rows
+
+    fs_version_rows = queryset.exclude(fs_version=current_fs_version).filter(Q(retry_at__isnull=True) | Q(retry_at__gt=now))
+    stale_batch = []
+
+    def queue_stale_fs_batch() -> None:
+        if not stale_batch:
+            return
         if wait_for_turn:
             wait_for_turn()
-        batch = list(islice(remaining, batch_size))
-        if not batch:
-            break
         now = timezone.now()
+        # Do not bump fs_version here. The orchestrator calls Snapshot.save(),
+        # which performs the idempotent filesystem migration and commits the new
+        # fs_version in the same serialized worker path as normal crawls.
         updated = 0
-        for snapshot in batch:
-            if migrate:
-                from archivebox.services.runner import run_snapshot_maintenance
-
-                if wait_for_turn:
-                    wait_for_turn()
-                run_snapshot_maintenance(str(snapshot.id))
-                continue
+        for snapshot in stale_batch:
+            # Each row gets its own short autocommit UPDATE because this scan
+            # can touch millions of snapshots while a server is also alive.
+            # The modified_at predicate is the CAS guard: if the runner or
+            # admin changed the snapshot after paged_iterator read it, skip it
+            # and let the newer state decide whether migration is still due.
             updated += int(
                 snapshot.safe_update(
-                    {"retry_at": now, "modified_at": now},
+                    {
+                        "retry_at": now,
+                        "modified_at": now,
+                    },
                     refresh=False,
                     extra_filter={"fs_version": snapshot.fs_version},
                 ),
             )
-        stats["processed"] += len(batch)
+        stats["processed"] += len(stale_batch)
         stats["updated_db"] += updated
         stats["queued"] += updated
-        print(f"    Queued {stats['queued']} stale filesystem snapshots so far...")
-    stats["snapshots"] = stats["processed"]
+        print(f"    [{stats['processed']}/{total}] Queued {updated} filesystem migrations for orchestrator...")
+        stale_batch.clear()
+
+    for snapshot in (
+        fs_version_rows.only("id", "crawl_id", "timestamp", "fs_version", "modified_at")
+        .order_by("-timestamp")
+        .paged_iterator(chunk_size=batch_size)
+    ):
+        try:
+            stale_batch.append(snapshot)
+            if len(stale_batch) >= batch_size:
+                queue_stale_fs_batch()
+        except KeyboardInterrupt as err:
+            err.archivebox_resume = snapshot.timestamp
+            raise
+    queue_stale_fs_batch()
+
+    now = timezone.now()
+    # Crawls with no open child snapshots are already finished. Seal them here
+    # instead of waking the foreground runner; otherwise migration/update can
+    # accidentally re-enter full crawl execution for historical rows.
+    stats["crawls_sealed"] = (
+        Crawl.objects.filter(
+            status__in=Crawl.RUNNABLE_STATES,
+        )
+        .exclude(
+            snapshot_set__status__in=Snapshot.OPEN_STATES,
+        )
+        .update(
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
+            modified_at=now,
+        )
+    )
+    stats["updated_db"] += stats["crawls_sealed"]
     return stats
 
 
@@ -701,7 +890,6 @@ def process_filtered_snapshots(
     resume: str | None,
     batch_size: int,
     queue_for_archiving: bool = True,
-    reverse: bool = False,
     wait_for_turn=None,
 ) -> dict[str, Any]:
     """Process snapshots matching filters (DB query only)."""
@@ -724,7 +912,6 @@ def process_filtered_snapshots(
         before=before,
         after=after,
         resume=resume,
-        reverse=reverse,
     )
 
     total = snapshots.count()
@@ -743,10 +930,8 @@ def process_filtered_snapshots(
             stats["snapshot_ids"].append(str(snapshot.id))
             update_values = {}
             updated = 0
-            if not queue_for_archiving and snapshot.fs_migration_needed:
-                from archivebox.services.runner import run_snapshot_maintenance
-
-                run_snapshot_maintenance(str(snapshot.id))
+            if not isinstance(snapshot.current_step, int):
+                update_values["current_step"] = 0
             if queue_for_archiving:
                 update_values.update(
                     {
@@ -832,9 +1017,10 @@ def print_index_stats(stats: dict[str, Any]) -> None:
 
     print(f"""
 [green]Search Reindex Complete[/green]
-  Missing index runs: {stats["processed"]}
-  Requested runs:    {stats.get("requested", 0)}
-  Indexed snapshots: {stats.get("reindexed", 0)}
+  Scanned rows:      {stats["processed"]}
+  Requested jobs:    {stats.get("requested", stats["queued"])}
+  Queued index jobs: {stats["queued"]}
+  Already queued:    {stats.get("skipped_queued", 0)}
 """)
 
 
@@ -844,12 +1030,6 @@ def print_index_stats(stats: dict[str, Any]) -> None:
 @click.option("--continuous", is_flag=True, help="Run continuously as background worker")
 @click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
 @click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state")
-@click.option("--reverse", is_flag=True, help="Process oldest first instead of newest first for explicit scans")
-@click.option(
-    "--rescan",
-    is_flag=True,
-    help="Scan all snapshot directories for orphans and metadata repairs (slow, opt-in, safe to restart)",
-)
 @snapshot_filter_options(default_filter_type="exact")
 @docstring(update.__doc__)
 def main(**kwargs):

archivebox/cli/archivebox_version.py

@@ -4,10 +4,7 @@ __package__ = "archivebox.cli"
 
 import sys
 import os
-import json
-import asyncio
 import platform
-import re
 from pathlib import Path
 from collections.abc import Iterable
 
@@ -32,7 +29,8 @@ def _format_binary_abspath(
 
     candidate_bases: tuple[tuple[Path, str], ...] = (
         (pwd, "./"),
-        (lib_dir, "ABXPKG_LIB_DIR/"),
+        (lib_dir, "LIB_DIR/"),
+        (Path(os.environ.get("LIB_DIR", "")), "LIB_DIR/") if os.environ.get("LIB_DIR") else (Path(), ""),
         (personas_dir, "PERSONAS_DIR/"),
         (home, "~/"),
     )
@@ -49,8 +47,8 @@ def _format_binary_abspath(
             relative_str = relative.as_posix()
             if prefix == "./":
                 return "." if not relative_str else f"./{relative_str}"
-            if prefix == "ABXPKG_LIB_DIR/":
-                return "ABXPKG_LIB_DIR" if not relative_str else f"ABXPKG_LIB_DIR/{relative_str}"
+            if prefix == "LIB_DIR/":
+                return "LIB_DIR" if not relative_str else f"LIB_DIR/{relative_str}"
             if prefix == "PERSONAS_DIR/":
                 return "PERSONAS_DIR" if not relative_str else f"PERSONAS_DIR/{relative_str}"
             return "~" if not relative_str else f"~/{relative_str}"
@@ -61,10 +59,10 @@ def _format_binary_abspath(
 def _render_binary_abspath(abspath: str):
     from rich.text import Text
 
-    if abspath.startswith("ABXPKG_LIB_DIR/"):
-        return Text.assemble(("ABXPKG_LIB_DIR", "bright_blue"), (abspath.removeprefix("ABXPKG_LIB_DIR"), "green"))
-    if abspath == "ABXPKG_LIB_DIR":
-        return Text("ABXPKG_LIB_DIR", style="bright_blue")
+    if abspath.startswith("LIB_DIR/"):
+        return Text.assemble(("LIB_DIR", "bright_blue"), (abspath.removeprefix("LIB_DIR"), "green"))
+    if abspath == "LIB_DIR":
+        return Text("LIB_DIR", style="bright_blue")
     if abspath.startswith("PERSONAS_DIR/"):
         return Text.assemble(("PERSONAS_DIR", "medium_purple"), (abspath.removeprefix("PERSONAS_DIR"), "green"))
     if abspath == "PERSONAS_DIR":
@@ -80,28 +78,10 @@ def _render_binary_abspath(abspath: str):
     return Text(abspath, style="green")
 
 
-def _binary_row_dedupe_key(
-    *,
-    display_name: str,
-    valid: bool,
-    version: str,
-    provider: str,
-    abspath: str,
-) -> tuple[str, str, str, str]:
-    if not valid:
-        return (display_name, "", "", "")
-    try:
-        resolved_abspath = Path(abspath).expanduser().resolve(strict=False).as_posix()
-    except Exception:
-        resolved_abspath = abspath
-    return (display_name, provider, version, resolved_abspath)
-
-
 @enforce_types
 def version(
     quiet: bool = False,
     binaries: Iterable[str] = (),
-    no_cache: bool = False,
 ) -> list[str]:
     """Print the ArchiveBox version, debug metadata, and installed dependency versions"""
 
@@ -114,15 +94,13 @@ def version(
 
     from rich.panel import Panel
     from rich.console import Console
-    from abx_dl.tables import binary_dependency_status, binary_dependency_table
 
     from archivebox.config import CONSTANTS
     from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
     from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID, IN_DOCKER
     from archivebox.config.paths import get_data_locations, get_code_locations
-    from archivebox.misc.checks import is_archivebox_source_root
     from archivebox.misc.logging_util import printable_folder_status
-    from archivebox.config.common import get_config, normalize_runtime_config
+    from archivebox.config.common import get_config
 
     console = Console()
     prnt = console.print
@@ -148,8 +126,7 @@ def version(
     )
 
     try:
-        data_locations = get_data_locations(config=config)
-        OUTPUT_IS_REMOTE_FS = data_locations.DATA_DIR.is_mount or data_locations.ARCHIVE_DIR.is_mount
+        OUTPUT_IS_REMOTE_FS = get_data_locations().DATA_DIR.is_mount or get_data_locations().ARCHIVE_DIR.is_mount
     except Exception:
         OUTPUT_IS_REMOTE_FS = False
 
@@ -177,38 +154,7 @@ def version(
     )
     prnt()
 
-    in_data_dir = (
-        not is_archivebox_source_root(CONSTANTS.DATA_DIR)
-        and os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK)
-        and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)
-    )
-    if isinstance(binaries, str):
-        requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
-    else:
-        requested_names = {name for name in (binaries or ()) if name}
-
-    def binary_is_requested(logical_name: str, actual_name: str, display_name: str) -> bool:
-        return not requested_names or bool({logical_name, actual_name, display_name} & requested_names)
-
-    def plugin_may_have_requested_binary(plugin) -> bool:
-        if not requested_names:
-            return True
-        aliases = {plugin.name}
-        for required_binary in plugin.config.required_binaries:
-            raw_name = str(required_binary.name)
-            aliases.add(raw_name)
-            template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", raw_name)
-            if template_match:
-                config_key = template_match.group(1)
-                aliases.add(config_key)
-                aliases.add(config_key.removesuffix("_BINARY"))
-                default = plugin.config.properties.get(config_key, {}).get("default")
-                if default:
-                    aliases.add(str(default))
-        normalized_aliases = {alias.lower() for alias in aliases}
-        return bool({name.lower() for name in requested_names} & normalized_aliases)
-
-    if not in_data_dir:
+    if not (os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
         PANEL_TEXT = "\n".join(
             (
                 "",
@@ -228,223 +174,100 @@ def version(
                 subtitle="Full version info is only available when inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
             ),
         )
-    prnt()
-
-    if not in_data_dir and not requested_names:
-        prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
-        prnt("", "[grey53]Dependency checks require an initialized collection DATA_DIR.[/grey53]")
-        prnt()
-        prnt("[deep_sky_blue3][i] Code locations:[/deep_sky_blue3]")
-        try:
-            for name, path in get_code_locations(config=config).items():
-                if isinstance(name, str) and isinstance(path, dict):
-                    prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
-        except Exception as e:
-            prnt(f"  [red]Error getting code locations: {e}[/red]")
-        prnt()
-        prnt("[red][i] Data locations:[/red] (not in a data directory)")
         prnt()
         return []
 
     prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
     failures = []
-    seen_failures: set[str] = set()
 
-    from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog
-    from abx_dl.config import get_required_binary_requests
-    from abx_dl.dependencies import resolve_binary_requests
-    from abx_dl.orchestrator import create_bus
-    from abxpkg.binary_service import BinaryEvent, BinaryService
+    # Setup Django before importing models
+    try:
+        from archivebox.config.django import setup_django
 
-    plugins = get_plugin_catalog()
-    enabled_plugin_names = set(get_enabled_plugins(config=config))
-    runtime_config = normalize_runtime_config(dict(config.items()), json_safe=False)
-    derived_config: dict[str, object] = {}
-    if in_data_dir:
-        try:
-            from archivebox.config.django import setup_django
+        setup_django()
 
-            setup_django()
+        from archivebox.machine.models import Machine, Binary
 
-            from archivebox.machine.models import Machine
+        machine = Machine.current()
 
-            machine = Machine.current()
-            derived_config = normalize_runtime_config(machine.config, json_safe=False)
+        if isinstance(binaries, str):
+            requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
+        else:
+            requested_names = {name for name in (binaries or ()) if name}
 
-        except Exception as e:
-            prnt()
-            prnt("", f"[yellow]Warning: Could not query collection machine config; resolving through abxpkg: {e}[/yellow]")
+        db_binaries: dict[str, Binary] = {}
+        for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
+            db_binaries.setdefault(binary.name, binary)
 
-    declared_binary_specs: dict[str, dict[str, object]] = {}
-    binary_records_by_plugin: dict[str, list[dict[str, object]]] = {}
-    for plugin_name, plugin in plugins.items():
-        if not plugin_may_have_requested_binary(plugin):
-            continue
-        plugin_requested = plugin_name.lower() in {name.lower() for name in requested_names}
-        binary_records = get_required_binary_requests(
-            plugin,
-            plugin.config.required_binaries,
-            overrides=runtime_config,
-            derived_overrides=derived_config,
-            run_output_dir=CONSTANTS.DATA_DIR,
-        )
-        # Render the exact requests resolved below. Rehydrating all schemas for
-        # display doubled config work, and could describe different requests if
-        # a config file changed while dependency resolution was running.
-        binary_records_by_plugin[plugin_name] = binary_records
-        for binary_record in binary_records:
-            actual_name = str(binary_record["name"])
-            logical_name = (
-                Path(actual_name).expanduser().name
-                if ("/" in actual_name or "\\" in actual_name or actual_name.startswith("~"))
-                else actual_name
-            )
-            display_name = logical_name
-            if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
-                continue
-            # Resolve through abxpkg, never the DB's last-known installation row.
-            # Its cache validates current paths, permissions, package fingerprints
-            # and runtime options. Forcing cold resolution here rehashed every
-            # large binary and reran every version subprocess on each diagnostic,
-            # even in an image that had already validated all of them at build.
-            # Keep an explicit full recheck for diagnosing a suspect installation.
-            binary_record["no_cache"] = no_cache
-            signature = json.dumps(binary_record, sort_keys=True, default=str)
-            declared_binary_specs.setdefault(signature, binary_record)
+        all_binary_names = sorted(requested_names or set(db_binaries.keys()))
 
-    loaded_binaries: dict[str, BinaryEvent | None] = {}
-    if declared_binary_specs:
-        binary_bus = create_bus(name="ArchiveBoxVersionBinaryCheck")
-        BinaryService(binary_bus, auto_install=False, lib_dir=config.ABXPKG_LIB_DIR)
+        if not all_binary_names:
+            prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
+        else:
+            any_available = False
+            compact_paths = console.is_terminal
+            for name in all_binary_names:
+                if requested_names and name not in requested_names:
+                    continue
 
-        async def resolve_declared_binaries() -> dict[str, BinaryEvent | None]:
-            try:
-                return await resolve_binary_requests(binary_bus, declared_binary_specs)
-            finally:
-                await binary_bus.wait_until_idle()
-                await binary_bus.destroy(clear=False)
-
-        loaded_binaries = asyncio.run(resolve_declared_binaries())
-
-    rows_by_binary: dict[str, dict] = {}
-    any_rows = False
-    any_available = False
-    compact_paths = console.is_terminal
-    for plugin_name, plugin in plugins.items():
-        if not plugin_may_have_requested_binary(plugin):
-            continue
-        plugin_requested = plugin_name.lower() in {name.lower() for name in requested_names}
-        plugin_enabled = plugin_name in enabled_plugin_names
-        disabled_by = ", ".join(
-            f"{dependency.enabled_key}=False"
-            for dependency in plugins.select([plugin_name]).values()
-            if not config.get(dependency.enabled_key, True)
-        )
-        binary_records = binary_records_by_plugin[plugin_name]
-        for binary_record in binary_records:
-            binary_record["no_cache"] = no_cache
-            actual_name = str(binary_record["name"])
-            logical_name = (
-                Path(actual_name).expanduser().name
-                if ("/" in actual_name or "\\" in actual_name or actual_name.startswith("~"))
-                else actual_name
-            )
-            display_name = logical_name
-            if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
-                continue
-
-            loaded = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
-            abspath = loaded.abspath if loaded is not None else ""
-            version_str = str(loaded.version or "unknown") if loaded is not None else "unknown"
-            provider = str(loaded.binprovider or "env") if loaded is not None else "env"
-            valid = loaded is not None and Path(abspath).is_file()
-
-            any_rows = True
-            if valid:
-                display_path = (
-                    _format_binary_abspath(
-                        abspath,
-                        pwd=Path.cwd(),
-                        lib_dir=config.ABXPKG_LIB_DIR,
-                        personas_dir=CONSTANTS.PERSONAS_DIR,
-                        home=Path.home(),
+                installed = db_binaries.get(name)
+                if installed and installed.is_valid:
+                    display_name = Path(name).expanduser().name if ("/" in name or name.startswith("~")) else name
+                    display_path = (
+                        _format_binary_abspath(
+                            installed.abspath,
+                            pwd=Path.cwd(),
+                            lib_dir=config.LIB_DIR,
+                            personas_dir=CONSTANTS.PERSONAS_DIR,
+                            home=Path.home(),
+                        )
+                        if compact_paths
+                        else installed.abspath
                     )
-                    if compact_paths
-                    else abspath
+                    rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
+                    version_str = (installed.version or "unknown")[:15]
+                    provider = (installed.binprovider or "env")[:8]
+                    prnt(
+                        "",
+                        "[green]√[/green]",
+                        "",
+                        display_name.ljust(18),
+                        version_str.ljust(16),
+                        provider.ljust(8),
+                        rendered_path,
+                        overflow="ignore",
+                        crop=False,
+                    )
+                    any_available = True
+                    continue
+
+                status = (
+                    "[grey53]not recorded[/grey53]" if name in requested_names and installed is None else "[grey53]not installed[/grey53]"
                 )
-                rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
-                any_available = True
-            else:
-                rendered_path = "[grey53]not installed[/grey53]"
-                if plugin_enabled and display_name not in seen_failures:
-                    failures.append(display_name)
-                    seen_failures.add(display_name)
+                prnt("", "[red]X[/red]", "", name.ljust(18), status, overflow="ignore", crop=False)
+                failures.append(name)
 
-            if not plugin_enabled:
-                rendered_path = f"[grey53]disabled by {disabled_by or 'PLUGINS configuration'}[/grey53]"
+            if not any_available:
+                prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
 
-            row = {
-                "plugin": plugin_name,
-                "status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
-                "binary": display_name,
-                "version": version_str if valid else "-",
-                "provider": provider if valid else "-",
-                "path": rendered_path,
-                "style": "" if plugin_enabled else "dim",
-                "enabled": plugin_enabled,
-                "valid": valid,
-                "plugins": [plugin_name],
-                "disabled_by": [disabled_by or "PLUGINS configuration"] if not plugin_enabled else [],
-            }
-            existing = rows_by_binary.get(display_name)
-            if existing is None:
-                rows_by_binary[display_name] = row
-            else:
-                consumers = list(dict.fromkeys([*existing["plugins"], plugin_name]))
-                disabled_reasons = list(dict.fromkeys([*existing["disabled_by"], *row["disabled_by"]]))
-                # Disabled consumers cannot mask an active requirement. If any
-                # active request fails (e.g. a version floor), keep that failure.
-                if plugin_enabled and (not existing["enabled"] or not valid):
-                    existing.update(row)
-                existing["plugins"] = consumers
-                existing["disabled_by"] = disabled_reasons
-                existing["plugin"] = ", ".join(consumers)
-                if not existing["enabled"]:
-                    existing["path"] = f"[grey53]disabled by {', '.join(disabled_reasons)}[/grey53]"
+        # Show hint if no binaries are installed yet
+        has_any_installed = Binary.objects.filter(machine=machine).exclude(abspath="").exists()
+        if not has_any_installed:
+            prnt()
+            prnt("", "[grey53]Run [green]archivebox install[/green] to detect and install dependencies.[/grey53]")
 
-    rows = list(rows_by_binary.values())
-
-    if console.is_terminal and console.width < 120:
-        from rich.text import Text
-
-        for row in rows:
-            heading = Text.from_markup(str(row["status"]))
-            heading.append(f" {row['binary']} {row['version']} ({row['provider']}) · {row['plugin']}")
-            prnt(heading, style=str(row["style"]))
-            path = row["path"]
-            prnt(Text.assemble("   ", path if isinstance(path, Text) else Text.from_markup(str(path))), style=str(row["style"]))
-    else:
-        table = binary_dependency_table(rows)
-        table.expand = False
-        table.title = None
-        table.box = None
-        for column in table.columns:
-            column.width = None
-            column.max_width = None
-            column.ratio = None
-        prnt(table)
-
-    if not any_rows:
-        prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")
-    elif not any_available:
-        prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
+    except Exception as e:
+        # Handle database errors gracefully (locked, missing, etc.)
+        prnt()
+        prnt("", f"[yellow]Warning: Could not query binaries from database: {e}[/yellow]")
+        prnt("", "[grey53]Run [green]archivebox init[/green] and [green]archivebox install[/green] to set up dependencies.[/grey53]")
 
     if not binaries:
         # Show code and data locations
         prnt()
         prnt("[deep_sky_blue3][i] Code locations:[/deep_sky_blue3]")
         try:
-            for name, path in get_code_locations(config=config).items():
+            for name, path in get_code_locations().items():
                 if isinstance(name, str) and isinstance(path, dict):
                     prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
         except Exception as e:
@@ -454,7 +277,7 @@ def version(
         if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
             prnt("[bright_yellow][i] Data locations:[/bright_yellow]")
             try:
-                for name, path in get_data_locations(config=config).items():
+                for name, path in get_data_locations().items():
                     if isinstance(name, str) and isinstance(path, dict):
                         prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
             except Exception as e:
@@ -489,11 +312,6 @@ def version(
     is_flag=True,
     help="Only print ArchiveBox version number and nothing else. (equivalent to archivebox --version)",
 )
-@click.option(
-    "--no-cache",
-    is_flag=True,
-    help="Reprobe binary versions and hashes instead of reusing validated provider caches.",
-)
 @click.option(
     "--binaries",
     "-b",
@@ -501,7 +319,9 @@ def version(
 )
 @docstring(version.__doc__)
 def main(**kwargs):
-    version(**kwargs)
+    failures = version(**kwargs)
+    if failures:
+        raise SystemExit(1)
 
 
 if __name__ == "__main__":

archivebox/cli/rescan.py

@@ -1,239 +0,0 @@
-"""Explicit discovery of portable snapshots; no checkpoint or default tree walk."""
-
-import json
-import os
-import uuid
-from itertools import groupby
-from pathlib import Path
-
-from django.contrib.auth import get_user_model
-from django.core.exceptions import ValidationError
-from django.db import IntegrityError
-from rich import print
-
-from archivebox.config import CONSTANTS
-from archivebox.core.models import ArchiveResult, Snapshot
-from archivebox.crawls.models import Crawl
-from archivebox.misc.util import parse_date
-
-
-def _directories(parent: Path):
-    # Names encode dates / UUIDv7 creation time. Newest first within each
-    # subtree, with bounded memory and no traversal into plugin payloads.
-    with os.scandir(parent) as entries:
-        paths = [Path(entry.path) for entry in entries if entry.is_dir(follow_symlinks=False)]
-    yield from sorted(paths, key=lambda path: path.name, reverse=True)
-
-
-def _snapshot_directories(reverse=False):
-    dates = []
-    for user in _directories(CONSTANTS.USERS_DIR):
-        root = user / CONSTANTS.SNAPSHOTS_DIR_NAME
-        if root.is_symlink() or not root.is_dir():
-            continue
-        dates.extend(_directories(root))
-    dates.sort(key=lambda path: path.name, reverse=not reverse)
-    for _, date_group in groupby(dates, key=lambda path: path.name):
-        directories = [directory for date in date_group for domain in _directories(date) for directory in _directories(domain)]
-        yield from sorted(directories, key=lambda path: path.name, reverse=not reverse)
-
-
-def _fields(record, names):
-    return {name: record[name] for name in names.split() if name in record}
-
-
-def _dates(record):
-    return {name: parse_date(record[name]) for name in ("created_at", "modified_at") if record.get(name)}
-
-
-def _restore_dates(obj, record):
-    dates = _dates(record)
-    if dates:
-        type(obj).objects.filter(pk=obj.pk).update(**dates)
-        for name, value in dates.items():
-            setattr(obj, name, value)
-
-
-def _reconcile_directory(directory: Path) -> str:
-    snapshot_id = uuid.UUID(directory.name)
-    snapshot = Snapshot.objects.select_related("crawl__created_by").filter(pk=snapshot_id).first()
-    index = directory / CONSTANTS.JSONL_INDEX_FILENAME
-    if index.is_symlink():
-        raise ValueError("metadata must not be a symlink")
-    if not index.exists():
-        if snapshot is None or Path(snapshot.output_dir) != directory or snapshot.status != "sealed":
-            raise ValueError("missing index.jsonl; cannot reconstruct an unknown or active snapshot")
-        snapshot.write_index_jsonl(output_dir=directory)
-        snapshot.reconcile_filesystem_links()
-        return "Repaired"
-
-    records = [json.loads(line) for line in index.read_text().splitlines() if line.strip()]
-    if any(not isinstance(record, dict) for record in records):
-        raise ValueError("metadata records must be JSON objects")
-    snapshots = [record for record in records if record.get("type") == "Snapshot"]
-    if len(snapshots) != 1:
-        raise ValueError("expected exactly one Snapshot record")
-    record = snapshots[0]
-    if uuid.UUID(record["id"]) != snapshot_id:
-        raise ValueError("snapshot ID conflicts with directory name")
-    if record.get("status") != "sealed":
-        raise ValueError("snapshot is not sealed; finish the source capture first")
-    if record.get("fs_version") not in {Snapshot._fs_current_version(), *Snapshot._FS_VERSION_MIGRATION_PATHS}:
-        raise ValueError("unsupported filesystem version")
-    crawl_id = uuid.UUID(record["crawl_id"])
-    owner = directory.parents[3].name
-    crawl_records = [item for item in records if item.get("type") == "Crawl" and uuid.UUID(item["id"]) == crawl_id]
-    crawl_record = crawl_records[0] if crawl_records else {}
-    if crawl_record.get("created_by", owner) != owner:
-        raise ValueError("crawl owner conflicts with directory path")
-    crawl = Crawl.objects.select_related("created_by").filter(pk=crawl_id).first()
-    imported_crawl = crawl is None
-    if crawl and crawl.created_by.username != owner:
-        raise ValueError("existing crawl ID belongs to a different owner")
-    if snapshot and (snapshot.url != record["url"] or snapshot.crawl_id != crawl_id or snapshot.timestamp != record["timestamp"]):
-        raise ValueError("existing snapshot ID has conflicting identity; refusing to overwrite")
-    if snapshot and Path(snapshot.output_dir) != directory:
-        raise ValueError("existing snapshot has a different storage path")
-
-    # Validate identities before inserting anything. Never turn an ID collision
-    # into a new ID, and never overwrite a newer destination row.
-    # write_index_jsonl retains historical hook records before appending the
-    # canonical DB rows. The final record for each hook is authoritative.
-    results = {(item.get("plugin"), item.get("hook_name", "")): item for item in records if item.get("type") == "ArchiveResult"}.values()
-    pending = []
-    for result in results:
-        if uuid.UUID(result["snapshot_id"]) != snapshot_id:
-            raise ValueError("ArchiveResult belongs to another snapshot")
-        # Legacy JSON conversion produced records without IDs. Recover these
-        # by hook identity while preserving strict collision checks for IDs.
-        if not result.get("id"):
-            if not ArchiveResult.objects.filter(
-                snapshot_id=snapshot_id,
-                plugin=result["plugin"],
-                hook_name=result.get("hook_name", ""),
-            ).exists():
-                pending.append(result)
-            continue
-        result_id = uuid.UUID(result["id"])
-        existing = ArchiveResult.objects.filter(pk=result_id).first()
-        identity = (snapshot_id, result["plugin"], result.get("hook_name", ""))
-        if existing:
-            if (existing.snapshot_id, existing.plugin, existing.hook_name) != identity:
-                raise ValueError("existing ArchiveResult ID has conflicting identity")
-        elif ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin=identity[1], hook_name=identity[2]).exists():
-            raise ValueError("existing hook has a different ArchiveResult ID")
-        else:
-            pending.append(result)
-
-    imported = snapshot is None
-    if imported:
-        if Snapshot.objects.filter(timestamp=record["timestamp"]).exists():
-            raise ValueError("timestamp belongs to another snapshot")
-        if Snapshot.objects.filter(crawl_id=crawl_id, url=record["url"]).exists():
-            raise ValueError("crawl URL belongs to another snapshot")
-        user, _ = get_user_model().objects.get_or_create(
-            username=owner,
-            defaults={"is_active": False, "password": "!"},
-        )
-        if crawl is None:
-            # Old exports contain only crawl_id. Keep that identity but report
-            # that absent crawl metadata cannot be reconstructed exactly.
-            if not crawl_record:
-                print(f"[yellow]    {directory}: no Crawl record; preserving ID with recovered metadata[/yellow]")
-            crawl = Crawl(
-                id=crawl_id,
-                created_by=user,
-                status="sealed",
-                retry_at=None,
-                **_fields(crawl_record, "urls max_depth config tags_str label"),
-            )
-            crawl.save(force_insert=True)
-            _restore_dates(crawl, crawl_record)
-        snapshot = Snapshot.create_from_directory(directory, crawl=crawl, preserve_identity=True)
-        if snapshot is None:
-            raise ValueError("invalid snapshot metadata")
-        snapshot.status = "sealed"
-        snapshot.retry_at = None
-        if "config" not in record:
-            snapshot.config = {"PERMISSIONS": "private"}
-        if Path(snapshot.get_storage_path_for_version(Snapshot._fs_current_version())) != directory:
-            raise ValueError("metadata does not match its owner/date/domain directory")
-        snapshot.save(force_insert=True)
-        _restore_dates(snapshot, record)
-
-    # Each insert is durable. An interruption after the Snapshot insert is not
-    # completion: the next pass still imports missing result IDs and tags.
-    for result in pending:
-        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=bool(result.get("id")))
-    if any(not result.get("id") for result in pending):
-        snapshot.hydrate_archiveresult_output_metadata(snapshot_dir=directory)
-        snapshot.write_index_jsonl(output_dir=directory)
-    tags_before = set(snapshot.tags.values_list("name", flat=True))
-    snapshot._merge_tags_from_index(record)
-    tags_changed = tags_before != set(snapshot.tags.values_list("name", flat=True))
-    link = Path(snapshot.crawl.output_dir) / "snapshots" / snapshot.extract_domain_from_url(snapshot.url) / str(snapshot.id)
-    links_ok = link.is_symlink() and link.resolve() == directory.resolve()
-    if not links_ok:
-        snapshot.reconcile_filesystem_links()
-    if snapshot.fs_migration_needed:
-        snapshot.migrate_filesystem_to_current_version(source_dir=directory)
-    if imported:
-        _restore_dates(snapshot, record)
-    if imported_crawl:
-        _restore_dates(crawl, crawl_record)
-    # Leave source metadata untouched, including process/binary records that
-    # are historical evidence rather than runnable destination process state.
-    return "Imported" if imported else "Repaired" if pending or tags_changed or not links_ok else "Unchanged"
-
-
-def reconcile_known_snapshots(snapshots, wait_for_turn=None):
-    """Explicit metadata / asset repair; normal updates never call this scan."""
-    for snapshot in snapshots.filter(status="sealed").paged_iterator():
-        if wait_for_turn:
-            wait_for_turn()
-        if snapshot.fs_migration_needed:
-            snapshot.migrate_filesystem_to_current_version()
-        directory = Path(snapshot.output_dir)
-        if not directory.is_dir():
-            # Canceling queued metadata can seal a row before the runner ever
-            # creates storage. There is nothing on disk to reconcile for it.
-            if snapshot.downloaded_at is None and not snapshot.archiveresult_set.exists():
-                continue
-            raise ValueError(f"Missing snapshot directory: {directory}")
-        _reconcile_directory(directory)
-        for result in snapshot.archiveresult_set.all():
-            result.update_output_metadata_from_filesystem(snapshot_dir=directory, full_scan=True)
-
-
-def rescan_snapshots(wait_for_turn=None, reverse=False, scan_legacy=None):
-    stats = dict.fromkeys(("Imported", "Repaired", "Unchanged", "Errors"), 0)
-    print("[*] Rescanning snapshot directories (O(N), orphans first; completed records are reused)...")
-    known = {str(pk) for pk in Snapshot.objects.values_list("pk", flat=True)}
-    # Two streaming passes avoid enumerating the entire archive before the
-    # first import. Rows imported before interruption move to the second pass
-    # next time; only one day's directory names are held in memory at a time.
-    for existing in (False, True):
-        if existing and scan_legacy:
-            scan_legacy()
-        for directory in _snapshot_directories(reverse=reverse):
-            if (directory.name.replace("-", "") in known) != existing:
-                continue
-            if wait_for_turn:
-                wait_for_turn()
-            try:
-                outcome = _reconcile_directory(directory)
-                if directory.name.replace("-", "") in known:
-                    for result in ArchiveResult.objects.filter(snapshot_id=uuid.UUID(directory.name)):
-                        result.update_output_metadata_from_filesystem(snapshot_dir=directory, full_scan=True)
-                stats[outcome] += 1
-                if outcome != "Unchanged":
-                    print(f"    {outcome}: {directory}")
-            except (OSError, ValueError, KeyError, TypeError, ValidationError, IntegrityError) as err:
-                stats["Errors"] += 1
-                print(f"[yellow]    Unresolved: {directory}: {err}[/yellow]")
-            if sum(stats.values()) % 100 == 0:
-                print("    " + ", ".join(f"{key}: {value}" for key, value in stats.items()))
-    print("    " + ", ".join(f"{key}: {value}" for key, value in stats.items()))
-    if stats["Errors"]:
-        raise SystemExit(1)
-    return stats

archivebox/config/collection.py

@@ -7,7 +7,7 @@ from collections.abc import Mapping
 from typing import Any
 
 from archivebox.config.constants import CONSTANTS
-from archivebox.config.configset import CaseConfigParser, decode_config_inputs
+from archivebox.config.configset import CaseConfigParser
 from archivebox.misc.logging import AttrDict
 
 
@@ -17,7 +17,7 @@ CONFIG_FILE_HEADER = (
     "# You can add options here manually in INI format, or automatically by running:\n"
     "#    archivebox config --set KEY=VALUE\n"
     "#\n"
-    "# This file is kept in sync 1:1 with the current Machine.config in the DB —\n"
+    "# This file is kept in sync 1:1 with Machine.config in the index DB —\n"
     "# editing either side propagates to the other. ``archivebox init`` reads\n"
     "# this file on startup; the admin Machine.config editor writes both.\n"
     "#\n"
@@ -160,7 +160,7 @@ def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
     so they round-trip through INI's string-only storage. When reading the
     file back into ``Machine.config`` (a JSONField that holds native types)
     those strings have to be decoded — otherwise downstream consumers like
-    ``MachineEvent`` → abx-dl see a JSON string
+    ``_emit_machine_config`` → ``MachineEvent`` → abx-dl see a JSON string
     where they expect a dict and raise ``TypeError``.
     Declared fields go through pydantic-settings' own ``field_is_complex`` /
     ``prepare_field_value`` so they're decoded per annotation. Undeclared
@@ -169,14 +169,31 @@ def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
     shape ``_coerce_to_str_dict`` writes them as.
     """
     from archivebox.config.common import ArchiveBoxConfig
+    from archivebox.config.configset import IniConfigSettingsSource
 
-    decoded = decode_config_inputs(ArchiveBoxConfig, file_config, decode_unknown_json=True)
-    if not str(decoded.get("BASE_URL") or "").strip():
-        from archivebox.config.common import base_url_from_legacy_server_config
-
-        legacy_base_url = base_url_from_legacy_server_config(decoded)
-        if legacy_base_url:
-            decoded["BASE_URL"] = legacy_base_url
+    decoder = IniConfigSettingsSource(ArchiveBoxConfig)
+    decoded: dict[str, Any] = dict(file_config)
+    declared_fields = set(ArchiveBoxConfig.model_fields)
+    for field_name, field in ArchiveBoxConfig.model_fields.items():
+        if field_name not in decoded:
+            continue
+        raw = decoded[field_name]
+        if not isinstance(raw, str) or not raw:
+            continue
+        if decoder.field_is_complex(field):
+            decoded[field_name] = decoder.prepare_field_value(field_name, field, raw, True)
+    for key, raw in list(decoded.items()):
+        if key in declared_fields:
+            continue
+        if not isinstance(raw, str) or not raw:
+            continue
+        first = raw[:1]
+        if first not in ("{", "["):
+            continue
+        try:
+            decoded[key] = json.loads(raw)
+        except (TypeError, ValueError):
+            continue
     return decoded
 
 
@@ -303,7 +320,7 @@ def write_config_file(config: dict[str, str]) -> AttrDict:
         # validate the updated_config by attempting to re-parse it
         from archivebox.config.common import get_config
 
-        updated_config = get_config(include_machine=False).as_dict()
+        updated_config = get_config().as_dict()
     except BaseException:  # lgtm [py/catch-base-exception]
         # something went horribly wrong, revert to the previous version
         with open(f"{config_path}.bak", encoding="utf-8") as old:

archivebox/config/common.py

@@ -2,57 +2,38 @@ from __future__ import annotations
 
 __package__ = "archivebox.config"
 
-import inspect
 import json
 import os
 import re
 import secrets
-import shutil
 import sys
+import shutil
+import inspect
+from functools import lru_cache
 from collections.abc import Mapping
 from datetime import timedelta
-from functools import cache, lru_cache
-from pathlib import Path
 from typing import Any, ClassVar, cast
-from urllib.parse import quote, urlparse
+from pathlib import Path
+from urllib.parse import quote
 
-from abx_plugins.plugins.base.utils import build_config_model
-from django.db import DatabaseError
-from pydantic import BaseModel, Field, PrivateAttr, TypeAdapter, create_model, field_validator, model_validator
-from pydantic_settings import BaseSettings, SettingsConfigDict
 from rich.console import Console
+from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
+from pydantic_settings import SettingsConfigDict
+from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
 
-from archivebox.config.configset import COMPUTED_CONFIG_KEYS, BaseConfigSet, decode_config_inputs
+from archivebox.config.configset import BaseConfigSet, IniConfigSettingsSource
+from archivebox.config.configset import COMPUTED_CONFIG_KEYS
 
 from .constants import CONSTANTS
 from .ldap import LDAPConfig
+from .version import get_COMMIT_HASH, get_BUILD_TIME, VERSION
 from .permissions import IN_DOCKER
-from .version import VERSION, get_BUILD_TIME, get_COMMIT_HASH
 
 ConfigOverrides = Mapping[str, object]
 ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
-
-def _is_cookie_file_config_key(key: str) -> bool:
-    return key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"} or key.endswith("_COOKIES_FILE")
-
-
-def _resolve_cookie_file_paths(config: dict[str, Any]) -> dict[str, Any]:
-    """Keep collection-relative auth paths valid after hooks change cwd."""
-    for key, value in config.items():
-        if _is_cookie_file_config_key(key):
-            if value is None or (isinstance(value, str) and not value.strip()):
-                config[key] = ""
-                continue
-            file_path = Path(value).expanduser()
-            if not file_path.is_absolute():
-                file_path = CONSTANTS.DATA_DIR / file_path
-            config[key] = str(file_path.absolute())
-    return config
-
-
 ###################### Config ##########################
 
 _STDOUT_CONSOLE = Console()
@@ -86,44 +67,6 @@ def permissions_from_legacy_public_flags(raw_config: Mapping[str, object]) -> st
     return None
 
 
-def base_url_from_legacy_server_config(raw_config: Mapping[str, object]) -> str:
-    """Derive the current canonical URL from the pre-0.9 server settings."""
-
-    def normalize(value: object) -> str:
-        raw = str(value or "").strip()
-        if not raw:
-            return ""
-        parsed = urlparse(raw if "://" in raw else f"http://{raw}")
-        return f"{parsed.scheme}://{parsed.netloc}" if parsed.netloc else ""
-
-    for key in ("ARCHIVE_BASE_URL", "ADMIN_BASE_URL"):
-        candidate = normalize(raw_config.get(key))
-        if candidate:
-            return candidate
-
-    listen_host = str(raw_config.get("LISTEN_HOST") or "").strip()
-    if listen_host and listen_host.lower() not in {"archivebox.localhost:8000", "archivebox.localhost:5797"}:
-        return normalize(listen_host)
-
-    csrf_origins = [normalize(origin) for origin in str(raw_config.get("CSRF_TRUSTED_ORIGINS") or "").split(",")]
-    csrf_origins = [origin for origin in csrf_origins if origin]
-    nondefault_csrf_origins = [
-        origin
-        for origin in csrf_origins
-        if origin not in {"http://admin.archivebox.localhost:8000", "http://admin.archivebox.localhost:5797"}
-    ]
-    if len(nondefault_csrf_origins) == 1:
-        return nondefault_csrf_origins[0]
-
-    allowed_hosts = [host.strip() for host in str(raw_config.get("ALLOWED_HOSTS") or "").split(",") if host.strip() != "*"]
-    if len(allowed_hosts) == 1:
-        return normalize(allowed_hosts[0])
-
-    if len(csrf_origins) == 1:
-        return csrf_origins[0]
-    return normalize(listen_host)
-
-
 def resolve_delete_after_config_value(*configs: Mapping[str, Any] | None) -> str:
     for config in configs:
         if config is None:
@@ -307,10 +250,10 @@ class StorageConfig(BaseConfigSet):
     # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
     TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
-    # ABXPKG_LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
+    # LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
     # must be able to contain executable binaries (up to 5GB size)
     # should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
-    ABXPKG_LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_ABXPKG_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
     OUTPUT_PERMISSIONS: str = Field(default="644")
     ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
@@ -329,7 +272,6 @@ class ServerConfig(BaseConfigSet):
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     SERVER_SECURITY_MODES: ClassVar[tuple[str, ...]] = (
-        "auto",
         "safe-subdomains-fullreplay",
         "safe-onedomain-nojsreplay",
         "unsafe-onedomain-noadmin",
@@ -337,11 +279,11 @@ class ServerConfig(BaseConfigSet):
     )
 
     SECRET_KEY: str = Field(default_factory=lambda: "".join(secrets.choice("abcdefghijklmnopqrstuvwxyz0123456789_") for _ in range(50)))
-    BIND_ADDR: str = Field(default="127.0.0.1:5797")
+    BIND_ADDR: str = Field(default="127.0.0.1:8000")
     BASE_URL: str = Field(default="")
     ALLOWED_HOSTS: str = Field(default="*")
     CSRF_TRUSTED_ORIGINS: str = Field(default="")
-    SERVER_SECURITY_MODE: str = Field(default="auto")
+    SERVER_SECURITY_MODE: str = Field(default="safe-subdomains-fullreplay")
 
     SNAPSHOTS_PER_PAGE: int = Field(default=50, ge=1)
     FOOTER_INFO: str = Field(
@@ -366,21 +308,11 @@ class ServerConfig(BaseConfigSet):
 
     @property
     def USES_SUBDOMAIN_ROUTING(self) -> bool:
-        if self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay":
-            return True
-        if self.SERVER_SECURITY_MODE != "auto":
-            return False
-
-        base_host = (urlparse(self.BASE_URL if "://" in self.BASE_URL else f"//{self.BASE_URL}").hostname or "").lower()
-        if base_host:
-            return base_host.endswith(".localhost")
-
-        bind_host = (urlparse(f"//{self.BIND_ADDR}").hostname or "").lower()
-        return bind_host in {"", "0.0.0.0", "127.0.0.1", "::", "::1", "localhost"}
+        return self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay"
 
     @property
     def ENABLES_FULL_JS_REPLAY(self) -> bool:
-        return self.USES_SUBDOMAIN_ROUTING or self.SERVER_SECURITY_MODE in (
+        return self.SERVER_SECURITY_MODE in (
             "safe-subdomains-fullreplay",
             "unsafe-onedomain-noadmin",
             "danger-onedomain-fullreplay",
@@ -396,7 +328,7 @@ class ServerConfig(BaseConfigSet):
 
     @property
     def SHOULD_NEUTER_RISKY_REPLAY(self) -> bool:
-        return self.SERVER_SECURITY_MODE in ("auto", "safe-onedomain-nojsreplay")
+        return self.SERVER_SECURITY_MODE == "safe-onedomain-nojsreplay"
 
     @property
     def IS_UNSAFE_MODE(self) -> bool:
@@ -414,52 +346,13 @@ class ServerConfig(BaseConfigSet):
         )
 
 
-@lru_cache(maxsize=16)
-def _default_sqlite_journal_mode(database_name: str) -> str:
-    if not IN_DOCKER:
-        return "WAL"
-
-    # Native Linux bind mounts share the kernel's locks and mmap; Docker alone
-    # does not make WAL unsafe. DELETE there lets any slow reader block tiny
-    # writes. Desktop VM shares (virtiofs/9p/FUSE), network filesystems and
-    # unknown mounts keep rollback journaling: their host-side readers may not
-    # share SQLite's -shm locking domain. Inspect the DB mount, not archive/.
-    database_path = Path(database_name).expanduser().resolve()
-    try:
-        mounts = Path("/proc/self/mountinfo").read_text().splitlines()
-    except OSError:
-        return "DELETE"
-    filesystem = ""
-    mount_depth = -1
-    for line in mounts:
-        fields, separator, details = line.partition(" - ")
-        if not separator:
-            continue
-        mount_path = fields.split()[4]
-        mount_path = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), mount_path)
-        mount = Path(mount_path)
-        if database_path.is_relative_to(mount) and len(mount.parts) > mount_depth:
-            filesystem = details.split()[0]
-            mount_depth = len(mount.parts)
-    return "WAL" if filesystem in {"ext2", "ext3", "ext4", "xfs", "btrfs", "zfs", "overlay", "tmpfs", "f2fs"} else "DELETE"
-
-
 class DatabaseConfig(BaseConfigSet):
     toml_section_header: str = "DATABASE_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
-    DATABASE_ENGINE: str = Field(
-        default="sqlite",
-        alias="ARCHIVEBOX_DATABASE_ENGINE",
-        pattern=r"(?i)^(sqlite|postgres|postgresql)$",
-    )
     DATABASE_NAME: str = Field(default=str(CONSTANTS.DATABASE_FILE), alias="ARCHIVEBOX_DATABASE_NAME")
-    DATABASE_HOST: str = Field(default="127.0.0.1", alias="ARCHIVEBOX_DATABASE_HOST")
-    DATABASE_PORT: int = Field(default=5432, alias="ARCHIVEBOX_DATABASE_PORT", ge=1, le=65535)
-    DATABASE_USER: str = Field(default="archivebox", alias="ARCHIVEBOX_DATABASE_USER")
-    DATABASE_PASSWORD: str = Field(default="", alias="ARCHIVEBOX_DATABASE_PASSWORD")
     SQLITE_JOURNAL_MODE: str = Field(
-        default_factory=lambda data: _default_sqlite_journal_mode(str(data["DATABASE_NAME"])),
+        default="WAL",
         alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
         pattern=r"(?i)^(DELETE|TRUNCATE|PERSIST|MEMORY|WAL|OFF)$",
     )
@@ -472,20 +365,6 @@ class DatabaseConfig(BaseConfigSet):
     SQLITE_LOCK_RETRY_TIMEOUT: float = Field(default=60.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_TIMEOUT", ge=0)
     SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0)
 
-    @model_validator(mode="after")
-    def reject_shared_filesystem_sqlite_wal(self):
-        if (
-            IN_DOCKER
-            and self.DATABASE_ENGINE.lower() == "sqlite"
-            and self.SQLITE_JOURNAL_MODE.upper() == "WAL"
-            and _default_sqlite_journal_mode(self.DATABASE_NAME) != "WAL"
-        ):
-            raise ValueError(
-                "SQLITE_JOURNAL_MODE=WAL requires a local filesystem with shared SQLite locking; "
-                "keep the database on a local Docker volume, use DELETE for host/VM shared mounts, or use PostgreSQL",
-            )
-        return self
-
 
 class ArchivingConfig(BaseConfigSet):
     toml_section_header: str = "ARCHIVING_CONFIG"
@@ -495,7 +374,6 @@ class ArchivingConfig(BaseConfigSet):
         default="",
         description="Comma-separated plugin selection for this run. Empty means use enabled plugin defaults.",
     )
-    PARSER: str = Field(default="auto", description="Parser used to read the submitted Crawl input document.")
 
     ONLY_NEW: bool = Field(default=True)
     INDEX_ONLY: bool = Field(default=False)
@@ -515,21 +393,9 @@ class ArchivingConfig(BaseConfigSet):
     USER_AGENT: str = Field(
         default=f"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)",
     )
-    COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-    AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    COOKIES_FILE: Path | None = Field(default=None)
 
-    @field_validator("COOKIES_FILE", "AUTH_STORAGE_FILE", mode="before")
-    @classmethod
-    def empty_cookie_file_is_unset(cls, value):
-        return None if isinstance(value, str) and not value.strip() else value
-
-    URL_DENYLIST: str = Field(
-        default=(
-            r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$"
-            r"|^https?://news\.ycombinator\.com/(?:vote|hide)/?(?:[?#]|$)"
-        ),
-        alias="URL_BLACKLIST",
-    )
+    URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
     URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")
 
     DEFAULT_PERSONA: str = Field(default="Default", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
@@ -553,7 +419,7 @@ class ArchivingConfig(BaseConfigSet):
             rprint("    (Setting it to somewhere between 30 and 3000 seconds is recommended)", file=sys.stderr)
             rprint(file=sys.stderr)
             rprint("    If you want to make ArchiveBox run faster, disable specific archive methods instead:", file=sys.stderr)
-            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#plugins", file=sys.stderr)
+            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles", file=sys.stderr)
             rprint(file=sys.stderr)
 
     @field_validator("CHECK_SSL_VALIDITY", mode="after")
@@ -626,7 +492,7 @@ class SearchBackendConfig(BaseConfigSet):
     toml_section_header: str = "SEARCH_BACKEND_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
-    SEARCH_BACKEND_ENGINE: str = Field(default="sonic", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
 
 def _plugin_user_config_value(value: Any) -> str:
@@ -641,36 +507,14 @@ def _plugin_user_config(config: Mapping[str, object]) -> dict[str, str]:
     return {key: _plugin_user_config_value(value) for key, value in config.items()}
 
 
-def _plugin_input_config(config: Mapping[str, object]) -> dict[str, object]:
-    """Return raw config inputs understood by core or plugin schemas."""
-    input_names = _archivebox_config_input_names()
-    return {str(key): value for key, value in config.items() if str(key) in input_names}
-
-
-def _explicit_plugin_enabled_keys(config: Mapping[str, object]) -> set[str]:
-    """Return canonical ``*_ENABLED`` keys explicitly provided by the user.
-
-    ``PLUGINS`` is a selector, not an override for concrete plugin enable flags.
-    If a user passes ``PLUGINS=wget`` and ``SAVE_WGET=False`` together, the
-    alias-resolved ``WGET_ENABLED=False`` must survive the model validator that
-    expands ``PLUGINS`` into concrete booleans.
-    """
-    provided = {str(key) for key in config}
-    explicit: set[str] = set()
-    for key, prop in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).items():
-        if not key.endswith("_ENABLED") or not isinstance(prop, Mapping):
-            continue
-        aliases = prop.get("x-aliases") if isinstance(prop, Mapping) else None
-        input_names = {key, *(str(alias) for alias in aliases if str(alias).strip())} if isinstance(aliases, list) else {key}
-        if provided & input_names:
-            explicit.add(key)
-    return explicit
-
-
 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
-    from archivebox.plugins.discovery import get_plugin_config_resolver
+    from archivebox.plugins.discovery import discover_plugin_configs
 
-    return get_plugin_config_resolver().schemas
+    schemas: PluginSchemaDocuments = {}
+    if BASE_CONFIG_PATH.exists():
+        schemas["base"] = json.loads(BASE_CONFIG_PATH.read_text())
+    schemas.update(discover_plugin_configs())
+    return schemas
 
 
 def _plugin_config_properties(plugin_schemas: PluginSchemaDocuments) -> dict[str, dict[str, Any]]:
@@ -692,10 +536,6 @@ def _archivebox_config_input_names() -> set[str]:
     for field in ArchiveBoxConfig.model_fields.values():
         if isinstance(field.alias, str):
             names.add(field.alias)
-    for prop in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).values():
-        aliases = prop.get("x-aliases") if isinstance(prop, Mapping) else None
-        if isinstance(aliases, list):
-            names.update(str(alias) for alias in aliases if str(alias).strip())
     return names
 
 
@@ -764,7 +604,7 @@ class ArchiveBoxBaseConfig(
     @classmethod
     def _plugin_field_scope(cls, key: str) -> str | None:
         scope = None
-        for schema in PLUGIN_CONFIG_SCHEMAS.values():
+        for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
             properties = schema.get("properties") if isinstance(schema, dict) else None
             if not isinstance(properties, dict) or key not in properties:
                 continue
@@ -776,7 +616,7 @@ class ArchiveBoxBaseConfig(
         return scope
 
     @classmethod
-    @cache
+    @lru_cache(maxsize=None)
     def scope_for_key(cls, key: str) -> str:
         for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
             properties = schema.get("properties") if isinstance(schema, dict) else None
@@ -807,10 +647,11 @@ class ArchiveBoxBaseConfig(
         runtime_derived_keys = {
             "ABX_INSTALL_CACHE",
             "ACTIVE_PERSONA",
+            "CHROME_DOWNLOADS_DIR",
+            "CHROME_USER_DATA_DIR",
             "CRAWL_DIR",
             "DEFAULT_PERSONA",
             "EXTRA_CONTEXT",
-            "PERSONAS_DIR",
             "SNAP_DIR",
         }
         return frozenset(
@@ -820,7 +661,7 @@ class ArchiveBoxBaseConfig(
     def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
         keys = type(self)._crawl_runtime_keys() if include_execution else type(self)._crawl_frozen_keys()
         payload = self.model_dump(mode="json")
-        return {key: payload[key] for key in keys if payload.get(key, None) is not None}
+        return {key: payload[key] for key in keys if payload.get(key) is not None}
 
     def for_crawl(self) -> dict[str, Any]:
         """Config scoped to crawl execution, without runtime object overlays."""
@@ -856,9 +697,9 @@ class ArchiveBoxBaseConfig(
         config["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
         scope_by_key = type(self)._scope_by_key()
         model_fields = type(self).model_fields
-        # ArchiveBox-only selectors are consumed during model validation to
-        # derive hook-owned *_ENABLED flags. They are stripped below after
-        # crawl/snapshot overlays have had a chance to apply.
+        # ArchiveBox owns SEARCH_BACKEND_ENGINE and uses it during model
+        # validation to derive the selected backend's *_ENABLED flag. Hooks
+        # only receive the backend-local flags, never the selector itself.
         config.pop("SEARCH_BACKEND_ENGINE", None)
         if persona is not None:
             for key, value in persona.get_derived_config().items():
@@ -880,13 +721,6 @@ class ArchiveBoxBaseConfig(
         if runtime_overrides:
             config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
 
-        _resolve_cookie_file_paths(config)
-
-        # Hooks should only see concrete plugin-local flags, never the
-        # ArchiveBox selectors used to derive them.
-        config.pop("PLUGINS", None)
-        config.pop("SEARCH_BACKEND_ENGINE", None)
-
         if extra_context:
             context: dict[str, Any] = {}
             if config.get("EXTRA_CONTEXT"):
@@ -901,51 +735,24 @@ class ArchiveBoxBaseConfig(
 
     @model_validator(mode="after")
     def resolve_runtime_paths(self):
-        lib_dir = self.ABXPKG_LIB_DIR.expanduser()
+        lib_dir = self.LIB_DIR.expanduser()
         if not lib_dir.is_absolute():
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
-        self.ABXPKG_LIB_DIR = lib_dir.resolve()
-
-        cookie_paths = {key: getattr(self, key) for key in type(self).model_fields if _is_cookie_file_config_key(key)}
-        for key, value in _resolve_cookie_file_paths(cookie_paths).items():
-            if key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"}:
-                setattr(self, key, Path(value) if value else None)
-            else:
-                setattr(self, key, value)
+        self.LIB_DIR = lib_dir.resolve()
 
         return self
 
     @model_validator(mode="after")
     def derive_plugin_enabled_config(self):
-        self._derive_plugin_enabled_config(respect_current_enabled=False)
-        return self
-
-    def _derive_plugin_enabled_config(self, *, respect_current_enabled: bool) -> None:
         plugin_names = _normalize_plugins_config_value(self.PLUGINS)
-        enabled_config_keys = _plugin_enabled_config_keys()
-        disabled_plugins = []
-        if respect_current_enabled:
-            disabled_plugins = [plugin_name for plugin_name, enabled_key in enabled_config_keys.items() if not getattr(self, enabled_key)]
-        selected_plugin_roots = plugin_names
-        if selected_plugin_roots:
-            from archivebox.plugins.discovery import get_plugin_catalog
-
-            selected_plugins = set(
-                get_plugin_catalog().select(
-                    sorted(selected_plugin_roots),
-                    disabled_names=disabled_plugins,
-                ),
-            )
-        else:
-            selected_plugins = set()
+        selected_plugins = _plugins_with_required_plugins(plugin_names) if plugin_names else set()
         search_backend = self.SEARCH_BACKEND_ENGINE.strip().lower()
         if search_backend:
             selected_plugins.add(f"search_backend_{search_backend}")
-        if respect_current_enabled:
-            selected_plugins.difference_update(disabled_plugins)
-        for plugin_name, enabled_key in enabled_config_keys.items():
+        for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
             if plugin_names or plugin_name in selected_plugins:
                 setattr(self, enabled_key, plugin_name in selected_plugins)
+        return self
 
 
 def _build_archivebox_config_model(plugin_schemas: PluginSchemaDocuments) -> type[ArchiveBoxBaseConfig]:
@@ -968,12 +775,6 @@ PLUGIN_CONFIG_SCHEMAS = _discover_plugin_config_schemas()
 ArchiveBoxConfig = _build_archivebox_config_model(PLUGIN_CONFIG_SCHEMAS)
 
 
-class ArchiveBoxSourceSettings(ArchiveBoxConfig, BaseSettings):
-    """Environment and INI source loader for the pure runtime config model."""
-
-    model_config = SettingsConfigDict(**{**ArchiveBoxConfig.model_config, "extra": "ignore"})
-
-
 def _normalize_plugins_config_value(value: Any) -> set[str]:
     if value is None:
         return set()
@@ -1008,6 +809,23 @@ def _plugin_enabled_config_keys() -> dict[str, str]:
     return enabled_keys
 
 
+def _plugins_with_required_plugins(plugin_names: set[str]) -> set[str]:
+    selected = set(plugin_names)
+    pending = list(selected)
+    while pending:
+        plugin_name = pending.pop()
+        schema = PLUGIN_CONFIG_SCHEMAS.get(plugin_name, {})
+        required_plugins = schema.get("required_plugins") if isinstance(schema, dict) else None
+        if not isinstance(required_plugins, list):
+            continue
+        for required_plugin in required_plugins:
+            required_plugin_name = str(required_plugin).strip().lower()
+            if required_plugin_name and required_plugin_name not in selected:
+                selected.add(required_plugin_name)
+                pending.append(required_plugin_name)
+    return selected
+
+
 def get_live_config_url(key: str) -> str:
     return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
 
@@ -1130,15 +948,9 @@ def get_request_config(request: Any, *, resolve_plugins: bool = False) -> Archiv
     request_config = request_state.get("archivebox_config")
     request_config_resolves_plugins = bool(request_state.get("_archivebox_config_resolves_plugins", False))
     if request_config is None or (resolve_plugins and not request_config_resolves_plugins):
-        from django.conf import settings
-
-        request_config = get_config(base_config=settings.CONFIG, resolve_plugins=resolve_plugins)
+        request_config = get_config(resolve_plugins=resolve_plugins)
+        request.archivebox_config = request_config
         request._archivebox_config_resolves_plugins = resolve_plugins
-    if request_config.SERVER_SECURITY_MODE == "auto":
-        request_host = (urlparse(f"//{request.get_host()}").hostname or "").lower().rstrip(".")
-        effective_mode = "safe-subdomains-fullreplay" if request_host.endswith(".localhost") else "safe-onedomain-nojsreplay"
-        request_config = request_config.model_copy(update={"SERVER_SECURITY_MODE": effective_mode})
-    request.archivebox_config = request_config
     return request_config
 
 
@@ -1177,23 +989,18 @@ def get_config(
             if apps.ready:
                 from archivebox.machine.models import Machine
 
-                machine = Machine.current_readonly()
-        except (DatabaseError, ImportError, RuntimeError, TypeError, ValueError):
+                machine = Machine.current()
+        except Exception:
             machine = None
 
     if persona is None and crawl is not None:
         persona = crawl.resolve_persona()
 
     config_data: ConfigPayload = dict(defaults or {})
-    config_data["PERSONAS_DIR"] = str(CONSTANTS.PERSONAS_DIR)
     base_config_payload: ConfigPayload = {}
-    file_config: dict[str, Any] | None = None
-    base_config_model = ArchiveBoxSourceSettings() if base_config is None else None
-
     if crawl_config_base:
-        assert base_config_model is not None
         config_data.update(
-            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
         )
         config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
     elif base_config is not None:
@@ -1203,32 +1010,21 @@ def get_config(
             base_config_payload.update(dict(base_config))
         config_data.update(normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False))
     else:
-        assert base_config_model is not None
         config_data.update(
-            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
         )
-        file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
-        legacy_config = {**file_config, **os.environ}
-        legacy_permissions = permissions_from_legacy_public_flags(legacy_config)
+        legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ})
         if legacy_permissions:
             config_data["PERMISSIONS"] = legacy_permissions
-        if not str(config_data.get("BASE_URL") or "").strip():
-            legacy_base_url = base_url_from_legacy_server_config(legacy_config)
-            if legacy_base_url:
-                config_data["BASE_URL"] = legacy_base_url
 
     scope_overrides: ConfigPayload = {}
 
     if include_machine and machine is not None and machine.config:
-        machine_config = machine.config
-        if resolve_plugins:
-            from archivebox.machine.models import _sanitize_machine_config
-
-            machine_config = _sanitize_machine_config(machine_config, lib_dir=config_data.get("ABXPKG_LIB_DIR"))
+        from archivebox.machine.models import _sanitize_machine_config
 
         scope_overrides.update(
             normalize_runtime_config(
-                machine_config,
+                _sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")),
                 only_crawl_execution=crawl_config_base,
                 exclude_runtime_derived=True,
                 json_safe=False,
@@ -1267,34 +1063,26 @@ def get_config(
     archivebox_scope_overrides = {key: value for key, value in scope_overrides.items() if key in _archivebox_config_input_names()}
     config_data.update(archivebox_scope_overrides)
 
-    explicit_plugin_enabled_keys: set[str] = set()
     if resolve_plugins:
+        plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
-        crawl_selected_plugins = crawl_config_base and bool(_normalize_plugins_config_value(dict(crawl.config or {}).get("PLUGINS")))
-        # A frozen crawl-level PLUGINS selector is the exact extractor set for
-        # this crawl. Env/file *_ENABLED values are ambient defaults and must
-        # not be re-applied after validation, or a later process env can change
-        # which hooks an already-queued crawl runs.
-        if not crawl_selected_plugins:
-            explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(os.environ))
-        explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(scope_overrides))
         plugin_user_config = _plugin_user_config(
             {
-                **_plugin_input_config(os.environ),
+                **normalize_runtime_config(config_data, only_crawl_execution=True, json_safe=False),
                 **scope_overrides,
             },
         )
         if not crawl_config_base:
-            if file_config is None:
-                file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
-            explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(file_config))
             plugin_user_config = {
-                **_plugin_user_config(_plugin_input_config(file_config)),
+                **normalize_runtime_config(
+                    BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE),
+                    exclude_runtime_derived=True,
+                    json_safe=False,
+                ),
                 **plugin_user_config,
             }
-        from archivebox.plugins.discovery import get_plugin_config_resolver
-
-        plugin_sections = get_plugin_config_resolver().resolve(
+        plugin_sections = resolve_plugin_configs(
+            plugin_schemas,
             global_config=plugin_global_config,
             user_config=plugin_user_config,
             environ={},
@@ -1315,31 +1103,38 @@ def get_config(
         if crawl_config_base:
             config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
         config_data.update(archivebox_scope_overrides)
-        if crawl_selected_plugins:
-            # Crawl PLUGINS is a frozen selector. Drop ambient enable flags from
-            # env/file/defaults before model validation so the selected plugin
-            # set is derived from the crawl, not from the worker process that
-            # happens to resume it.
-            for enabled_key in _plugin_enabled_config_keys().values():
-                config_data.pop(enabled_key, None)
 
-    config_data = decode_config_inputs(ArchiveBoxConfig, config_data)
+    # Decode JSON-encoded complex values (dict/list fields) that came from
+    # string-only sources before validation. ``IniConfigSettingsSource`` does
+    # this for the ArchiveBox.conf path, but Machine.config (mirrored from the
+    # INI via ``_coerce_to_str_dict``) and plugin/env scope overrides bypass
+    # pydantic-settings sources entirely — they feed JSON strings directly
+    # into ``model_validate``, which rejects ``"{...}"`` for a ``dict[str, str]``
+    # field. Run pydantic-settings' own complex-value decoder here so every
+    # source converges on the same shape before validation.
+    _complex_decoder = IniConfigSettingsSource(ArchiveBoxConfig)
+    for _field_name, _field in ArchiveBoxConfig.model_fields.items():
+        if _field_name not in config_data:
+            continue
+        _raw = config_data[_field_name]
+        if not isinstance(_raw, str) or not _raw:
+            continue
+        if _complex_decoder.field_is_complex(_field):
+            config_data[_field_name] = _complex_decoder.prepare_field_value(
+                _field_name,
+                _field,
+                _raw,
+                True,
+            )
 
     config = ArchiveBoxConfig.model_validate(config_data)
-    if config.SERVER_SECURITY_MODE == "auto":
-        base_host = (urlparse(config.BASE_URL if "://" in config.BASE_URL else f"//{config.BASE_URL}").hostname or "").lower()
-        if base_host.endswith(".localhost"):
-            config = config.model_copy(update={"SERVER_SECURITY_MODE": "safe-subdomains-fullreplay"})
-    for key in explicit_plugin_enabled_keys:
-        if key in config_data:
-            setattr(config, key, TypeAdapter(bool).validate_python(config_data[key]))
-    if config.PLUGINS:
-        config._derive_plugin_enabled_config(respect_current_enabled=True)
     if redact_sensitive:
         for key in type(config).model_fields:
             value = config[key]
             if is_sensitive_config_key(key) and value not in (None, ""):
                 setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
+    os.environ["LIB_DIR"] = str(config.LIB_DIR)
+    os.environ["ABXPKG_LIB_DIR"] = str(config.LIB_DIR)
     archiving_warning_key = (config.TIMEOUT, config.USE_COLOR)
     if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS:
         config.warn_if_invalid()
@@ -1349,14 +1144,13 @@ def get_config(
 
 def get_all_configs() -> dict[str, BaseConfigSet]:
     """Get all config section objects as a dictionary."""
-    resolved = ArchiveBoxSourceSettings().model_dump(mode="json")
     return {
-        "SHELL_CONFIG": ShellConfig.model_validate(resolved),
-        "STORAGE_CONFIG": StorageConfig.model_validate(resolved),
-        "GENERAL_CONFIG": GeneralConfig.model_validate(resolved),
-        "SERVER_CONFIG": ServerConfig.model_validate(resolved),
-        "DATABASE_CONFIG": DatabaseConfig.model_validate(resolved),
-        "ARCHIVING_CONFIG": ArchivingConfig.model_validate(resolved),
-        "SEARCH_BACKEND_CONFIG": SearchBackendConfig.model_validate(resolved),
-        "LDAP_CONFIG": LDAPConfig.model_validate(resolved),
+        "SHELL_CONFIG": ShellConfig(),
+        "STORAGE_CONFIG": StorageConfig(),
+        "GENERAL_CONFIG": GeneralConfig(),
+        "SERVER_CONFIG": ServerConfig(),
+        "DATABASE_CONFIG": DatabaseConfig(),
+        "ARCHIVING_CONFIG": ArchivingConfig(),
+        "SEARCH_BACKEND_CONFIG": SearchBackendConfig(),
+        "LDAP_CONFIG": LDAPConfig(),
     }

archivebox/config/configset.py

@@ -2,13 +2,10 @@
 
 __package__ = "archivebox.config"
 
-import json
-from collections.abc import Mapping
-from configparser import ConfigParser
 from pathlib import Path
 from typing import Any, ClassVar
+from configparser import ConfigParser
 
-from pydantic import BaseModel
 from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict
 
 COMPUTED_CONFIG_KEYS = (
@@ -33,17 +30,31 @@ class CaseConfigParser(ConfigParser):
         return optionstr
 
 
-def read_ini_config(config_path: str | Path) -> dict[str, Any]:
-    """Read and flatten an ArchiveBox INI config file."""
-    config_path = Path(config_path)
+# ``IniConfigSettingsSource.get_field_value`` is called once per pydantic field
+# (hundreds of fields across the GlobalConfig + plugin configs). Without
+# caching, every field re-opens and re-parses ``ArchiveBox.conf`` from disk —
+# that's the dominant cost of ``get_config()`` (each call was ~130ms; profile
+# showed 632 ``parser.read(config_path)`` invocations per call). The cache is
+# keyed on (path, mtime) so external edits to the file still get picked up.
+_INI_CACHE: dict[tuple[str, float], dict[str, Any]] = {}
+
+
+def _read_ini_config_cached(config_path_str: str) -> dict[str, Any]:
+    config_path = Path(config_path_str)
     try:
-        if not config_path.exists():
-            return {}
-        parser = CaseConfigParser(interpolation=None)
-        parser.read(config_path)
-    except OSError:
+        mtime = config_path.stat().st_mtime
+    except FileNotFoundError:
         return {}
-    return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+    cache_key = (str(config_path), mtime)
+    cached = _INI_CACHE.get(cache_key)
+    if cached is not None:
+        return cached
+    parser = CaseConfigParser()
+    parser.read(config_path)
+    flat = {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
+    _INI_CACHE.clear()
+    _INI_CACHE[cache_key] = flat
+    return flat
 
 
 class IniConfigSettingsSource(PydanticBaseSettingsSource):
@@ -64,7 +75,18 @@ class IniConfigSettingsSource(PydanticBaseSettingsSource):
         return field_value, field_name, value_is_complex
 
     def __call__(self) -> dict[str, Any]:
-        return decode_config_inputs(self.settings_cls, self._load_config_file())
+        # Use the per-field path (``get_field_value`` + ``prepare_field_value``)
+        # so complex types get JSON-decoded. The previous flat-dict return
+        # skipped pydantic-settings' complex-value handling entirely.
+        result: dict[str, Any] = {}
+        for field_name, field in self.settings_cls.model_fields.items():
+            value, key, value_is_complex = self.get_field_value(field, field_name)
+            if value is None:
+                continue
+            prepared = self.prepare_field_value(field_name, field, value, value_is_complex)
+            if prepared is not None:
+                result[key] = prepared
+        return result
 
     def _load_config_file(self) -> dict[str, Any]:
         try:
@@ -74,42 +96,14 @@ class IniConfigSettingsSource(PydanticBaseSettingsSource):
         except ImportError:
             return {}
 
-        return read_ini_config(config_path)
+        return _read_ini_config_cached(str(config_path))
 
 
-def decode_config_inputs(
-    settings_cls: type[BaseSettings],
-    config: Mapping[str, Any],
-    *,
-    decode_unknown_json: bool = False,
-) -> dict[str, Any]:
-    """Decode string-only config values once at the boundary to typed config."""
-    decoder = IniConfigSettingsSource(settings_cls)
-    decoded: dict[str, Any] = dict(config)
-    for source_key, raw in list(decoded.items()):
-        if not isinstance(raw, str) or not raw:
-            continue
-        field_name = source_key if source_key in settings_cls.model_fields else source_key.upper()
-        field = settings_cls.model_fields.get(field_name)
-        if field is None:
-            if decode_unknown_json and raw[:1] in ("{", "["):
-                try:
-                    decoded[source_key] = json.loads(raw)
-                except (TypeError, ValueError):
-                    pass
-            continue
-        if decoder.field_is_complex(field):
-            decoded[field_name] = decoder.prepare_field_value(field_name, field, raw, True)
-            if source_key != field_name:
-                decoded.pop(source_key, None)
-    return decoded
-
-
-class BaseConfigSet(BaseModel):
+class BaseConfigSet(BaseSettings):
     """
-    Pure typed runtime model for config sections.
+    Base class for config sections.
 
-    Source-loading subclasses combine this model with ``BaseSettings`` and load:
+    Automatically loads values from (highest to lowest priority):
     1. Environment variables
     2. ArchiveBox.conf file (INI format, flattened)
     3. Default values
@@ -152,7 +146,14 @@ class BaseConfigSet(BaseModel):
     @classmethod
     def load_from_file(cls, config_path: Path) -> dict[str, str]:
         """Load config values from INI file."""
-        return read_ini_config(config_path)
+        if not config_path.exists():
+            return {}
+
+        parser = CaseConfigParser()
+        parser.read(config_path)
+
+        # Flatten all sections into single namespace
+        return {key.upper(): value for section in parser.sections() for key, value in parser.items(section)}
 
     def __getitem__(self, key: str) -> Any:
         if key in type(self).model_fields:

archivebox/config/constants.py

@@ -113,9 +113,10 @@ class ConstantsDict:
     TMP_DIR_NAME: str = "tmp"
     DEFAULT_TMP_DIR: Path = DATA_DIR / TMP_DIR_NAME / MACHINE_ID  # ./data/tmp/abc3244323
 
-    DEFAULT_ABXPKG_LIB_DIR: Path = _env_path(
-        "ABXPKG_LIB_DIR",
-        user_config_path("abx") / "lib",
+    LIB_DIR_NAME: str = "lib"
+    DEFAULT_LIB_DIR: Path = _env_path(
+        "LIB_DIR",
+        user_config_path("abx") / LIB_DIR_NAME,
     )
 
     RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset(
@@ -239,7 +240,7 @@ class ConstantsDict:
             SOURCES_DIR_NAME,
             LOGS_DIR_NAME,
             CACHE_DIR_NAME,
-            "lib",
+            LIB_DIR_NAME,
             TMP_DIR_NAME,
             PERSONAS_DIR_NAME,
             CUSTOM_TEMPLATES_DIR_NAME,
@@ -293,3 +294,5 @@ CONSTANTS = ConstantsDict
 CONSTANTS_CONFIG = AttrDict({key: value for key, value in CONSTANTS.__dict__.items() if key.isupper() and not key.startswith("_")})
 
 # add all key: values to globals() for easier importing, e.g.:
+# from archivebox.config.constants import IS_ROOT, PERSONAS_DIR, ...
+# globals().update(CONSTANTS)

archivebox/config/django.py

@@ -2,20 +2,21 @@ __package__ = "archivebox.config"
 
 import os
 import sys
-from datetime import UTC, datetime
+import subprocess
+
+from datetime import datetime, timezone
+
+from rich.console import Console
 
 import django
 import django.db
-from django.core.exceptions import ImproperlyConfigured
-from rich.console import Console
 
 from archivebox.misc import logging
 
-from .common import get_config
 from .constants import CONSTANTS
+from .common import get_config
 
 CONFIG = get_config()
-os.environ.setdefault("ABXPKG_LIB_DIR", str(CONFIG.ABXPKG_LIB_DIR))
 
 if not CONFIG.USE_COLOR:
     os.environ["NO_COLOR"] = "1"
@@ -30,47 +31,32 @@ logging.CONSOLE = CONSOLE
 DJANGO_SET_UP = False
 
 
-def setup_django(check_db=False) -> None:
+def setup_django(check_db=False, in_memory_db=False) -> None:
     from rich.panel import Panel
 
-    from archivebox.misc.checks import check_not_inside_source_dir
-
     global DJANGO_SET_UP
 
     if DJANGO_SET_UP:
+        # raise Exception('django is already set up!')
         # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
         return
 
-    check_not_inside_source_dir(CONSTANTS.DATA_DIR)
-
-    # SQLite creates index.sqlite3 during django.setup()/migrate. Apply the
-    # ArchiveBox file-mode policy before any DB connection can create the file,
-    # otherwise a permissive parent umask can expose a just-created DB until a
-    # later chmod runs.
-    os.umask(0o777 - (int(CONFIG.OUTPUT_PERMISSIONS, base=8) | 0o111))
-
     # Third-party patches are only needed once Django/apps are about to load.
     # Keeping them out of archivebox.__init__ avoids paying Django/Daphne setup
     # cost for cheap CLI startup paths like `archivebox <cmd> --help`.
     import archivebox.misc.monkey_patches  # noqa: F401
-    from archivebox.config.permissions import ARCHIVEBOX_GROUP, ARCHIVEBOX_USER, IS_ROOT, SudoPermission
+
+    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, SudoPermission
 
     # if running as root, chown the data dir to the archivebox user to make sure it's accessible to the archivebox user
     if IS_ROOT and ARCHIVEBOX_USER != 0:
         with SudoPermission(uid=0):
             # running as root is a special case where it's ok to be a bit slower
             # make sure data dir is always owned by the correct user
-            os.chown(CONSTANTS.DATA_DIR, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
+            subprocess.run(["chown", f"{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", str(CONSTANTS.DATA_DIR)], stderr=subprocess.DEVNULL)
             if CONSTANTS.DATA_DIR.exists():
                 for child in CONSTANTS.DATA_DIR.iterdir():
-                    try:
-                        os.chown(child, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
-                    except FileNotFoundError:
-                        # A concurrent SQLite transaction can remove its journal
-                        # between directory enumeration and chown. There is no
-                        # ownership left to fix; other permission errors must
-                        # still fail startup rather than being hidden.
-                        continue
+                    subprocess.run(["chown", f"{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", str(child)], stderr=subprocess.DEVNULL)
 
     # Suppress the "database access during app initialization" warning
     # This warning can be triggered during django.setup() but is safe to ignore
@@ -86,30 +72,40 @@ def setup_django(check_db=False) -> None:
     try:
         from django.core.management import call_command
 
-        # Initialize the configured file-based database without running
-        # migrations automatically; `archivebox init` owns migrations.
-        try:
-            django.setup()
-        # User config and import errors are reported through one CLI message.
-        except (ImproperlyConfigured, django.db.Error, ImportError, OSError, RuntimeError, ValueError) as e:
-            is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ("help", "version", "--help", "--version"))
-            if not is_using_meta_cmd:
-                # show error message to user only if they're not running a meta command / just trying to get help
-                STDERR.print()
-                STDERR.print(
-                    Panel(
-                        f"\n[red]{e.__class__.__name__}[/red]: [yellow]{e}[/yellow]\nPlease check your config and [blue]DATA_DIR[/blue] permissions.\n",
-                        title="\n\n[red][X] Error while trying to load database![/red]",
-                        subtitle="[grey53]NO WRITES CAN BE PERFORMED[/grey53]",
-                        expand=False,
-                        style="bold red",
-                    ),
-                )
-                STDERR.print()
-                import traceback
+        if in_memory_db:
+            raise Exception("dont use this anymore")
 
-                traceback.print_exc()
-            return
+            # some commands dont store a long-lived sqlite3 db file on disk.
+            # in those cases we create a temporary in-memory db and run the migrations
+            # immediately to get a usable in-memory-database at startup
+            os.environ.setdefault("ARCHIVEBOX_DATABASE_NAME", ":memory:")
+            django.setup()
+
+            call_command("migrate", interactive=False, verbosity=0)
+        else:
+            # Otherwise use default sqlite3 file-based database and initialize django
+            # without running migrations automatically (user runs them manually by calling init)
+            try:
+                django.setup()
+            except Exception as e:
+                is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ("help", "version", "--help", "--version"))
+                if not is_using_meta_cmd:
+                    # show error message to user only if they're not running a meta command / just trying to get help
+                    STDERR.print()
+                    STDERR.print(
+                        Panel(
+                            f"\n[red]{e.__class__.__name__}[/red]: [yellow]{e}[/yellow]\nPlease check your config and [blue]DATA_DIR[/blue] permissions.\n",
+                            title="\n\n[red][X] Error while trying to load database![/red]",
+                            subtitle="[grey53]NO WRITES CAN BE PERFORMED[/grey53]",
+                            expand=False,
+                            style="bold red",
+                        ),
+                    )
+                    STDERR.print()
+                    import traceback
+
+                    traceback.print_exc()
+                return
 
         from archivebox.core.settings_logging import ERROR_LOG as DEFAULT_ERROR_LOG
 
@@ -117,13 +113,13 @@ def setup_django(check_db=False) -> None:
         error_log = DEFAULT_ERROR_LOG
         with open(error_log, "a", encoding="utf-8") as f:
             command = " ".join(sys.argv)
-            ts = datetime.now(UTC).strftime("%Y-%m-%d__%H:%M:%S")
+            ts = datetime.now(timezone.utc).strftime("%Y-%m-%d__%H:%M:%S")
             config = get_config()
             f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={config.IN_DOCKER} IS_TTY={config.IS_TTY}\n")
 
         if check_db:
             # make sure the data dir is owned by a non-root user
-            if CONSTANTS.DATA_DIR.stat().st_uid == 0 and not (IS_ROOT and ARCHIVEBOX_USER == 0):
+            if CONSTANTS.DATA_DIR.stat().st_uid == 0:
                 STDERR.print("[red][X] Error: ArchiveBox DATA_DIR cannot be owned by root![/red]")
                 STDERR.print(f"    {CONSTANTS.DATA_DIR}")
                 STDERR.print()
@@ -149,14 +145,23 @@ def setup_django(check_db=False) -> None:
             for conn in connections.all():
                 conn.close_if_unusable_or_obsolete()
 
-            from archivebox.misc.db import database_display_location, database_exists
-
-            assert database_exists(), (
-                f"No database {database_display_location()} found for: {CONSTANTS.DATA_DIR} (Are you in an ArchiveBox collection directory?)"
+            sql_index_path = CONSTANTS.DATABASE_FILE
+            assert os.access(sql_index_path, os.F_OK), (
+                f"No database file {sql_index_path} found in: {CONSTANTS.DATA_DIR} (Are you in an ArchiveBox collection directory?)"
             )
 
+            # https://docs.pydantic.dev/logfire/integrations/django/ Logfire Debugging
+            # if settings.DEBUG_LOGFIRE:
+            #     from opentelemetry.instrumentation.sqlite3 import SQLite3Instrumentor
+            #     SQLite3Instrumentor().instrument()
+
+            #     import logfire
+
+            #     logfire.configure()
+            #     logfire.instrument_django(is_sql_commentor_enabled=True)
+            #     logfire.info(f'Started ArchiveBox v{CONSTANTS.VERSION}', argv=sys.argv)
+
     except KeyboardInterrupt:
-        DJANGO_SET_UP = False
         raise
 
     DJANGO_SET_UP = True

archivebox/config/paths.py

@@ -5,6 +5,7 @@ import socket
 import hashlib
 import tempfile
 import platform
+import subprocess
 from pathlib import Path
 from functools import cache
 from datetime import datetime
@@ -58,11 +59,9 @@ def _get_collection_id(DATA_DIR=DATA_DIR, force_create=False) -> str:
     collection_id = hashlib.sha256(f"{machine_id}:{collection_path}@{creation_date}".encode()).hexdigest()[:8]
 
     try:
-        # only persist collection_id file if this dir already looks like a real collection
-        # (has an index.sqlite3, or an ArchiveBox.conf when the DB lives in postgres),
+        # only persist collection_id file if we already have an index.sqlite3 file present
         # otherwise we might be running in a directory that is not a collection, no point creating cruft files
-        collection_marker = os.path.isfile(DATABASE_FILE) or os.path.isfile(DATA_DIR / "ArchiveBox.conf")
-        collection_is_active = collection_marker and os.path.isdir(ARCHIVE_DIR) and os.access(DATA_DIR, os.W_OK)
+        collection_is_active = os.path.isfile(DATABASE_FILE) and os.path.isdir(ARCHIVE_DIR) and os.access(DATA_DIR, os.W_OK)
         if collection_is_active or force_create:
             collection_id_file.write_text(collection_id)
 
@@ -70,9 +69,9 @@ def _get_collection_id(DATA_DIR=DATA_DIR, force_create=False) -> str:
             if IS_ROOT:
                 with SudoPermission(uid=0):
                     if ARCHIVEBOX_USER == 0:
-                        collection_id_file.chmod(0o777)
+                        subprocess.run(["chmod", "777", str(collection_id_file)])
                     else:
-                        os.chown(collection_id_file, ARCHIVEBOX_USER, -1)
+                        subprocess.run(["chown", str(ARCHIVEBOX_USER), str(collection_id_file)])
     except (OSError, FileNotFoundError, PermissionError):
         pass
     return collection_id
@@ -85,31 +84,23 @@ def get_collection_id(DATA_DIR=DATA_DIR) -> str:
 
 
 @cache
-def get_machine_id(length: int = 8) -> str:
-    """Get a machine ID, using the network identity when no OS ID is available."""
-    import machineid
+def get_machine_id() -> str:
+    """Get a short, stable, unique ID for the current machine (e.g. abc45678)"""
 
+    MACHINE_ID = "unknown"
     try:
-        return machineid.hashed_id("archivebox")[:length]
-    except machineid.MachineIdNotFound:
-        import uuid
+        import machineid
 
-        import psutil
+        MACHINE_ID = machineid.hashed_id("archivebox")[:8]
+    except Exception:
+        try:
+            import uuid
+            import hashlib
 
-        node = uuid.getnode()
-        identity = str(node)
-        if node & (1 << 40):
-            # uuid.getnode() silently generates a per-process random value when
-            # tools such as ip/ifconfig are absent. Read interfaces directly so
-            # separate CLI processes agree even in a minimal container.
-            addresses = sorted(
-                address.address
-                for interface in psutil.net_if_addrs().values()
-                for address in interface
-                if address.family == psutil.AF_LINK and address.address not in ("", "00:00:00:00:00:00")
-            )
-            identity = str(int(addresses[0].replace(":", "").replace("-", ""), 16)) if addresses else f"hostname:{socket.gethostname()}"
-        return hashlib.sha256(identity.encode()).hexdigest()[:length]
+            MACHINE_ID = hashlib.sha256(str(uuid.getnode()).encode()).hexdigest()[:8]
+        except Exception:
+            pass
+    return MACHINE_ID
 
 
 @cache
@@ -138,7 +129,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
         if chown:
             # try fixing it using sudo permissions
             with SudoPermission(uid=uid, fallback=fallback):
-                os.chown(dir_path, uid, gid)
+                subprocess.run(["chown", f"{uid}:{gid}", str(dir_path)], stderr=subprocess.DEVNULL)
             return dir_is_writable(dir_path, uid=uid, gid=gid, fallback=fallback, chown=False)
     return False
 
@@ -230,7 +221,7 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxC
             create_and_chown_dir(candidate)
         except Exception:
             pass
-        if check_tmp_dir(candidate, throw=False, quiet=True, must_exist=True, config=config):
+        if check_tmp_dir(candidate, throw=False, quiet=True, must_exist=True):
             if autofix and config.TMP_DIR != candidate:
                 os.environ["TMP_DIR"] = str(candidate)
             return candidate
@@ -263,39 +254,21 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
 
     config = config or get_config(**config_kwargs)
 
-    # ABXPKG_LIB_DIR is either the shared platformdirs default or an explicit env/config override.
-    CANDIDATES = [config.ABXPKG_LIB_DIR]
+    # LIB_DIR is either the shared platformdirs default or an explicit env/config override.
+    CANDIDATES = [config.LIB_DIR]
 
     for candidate in CANDIDATES:
         try:
             create_and_chown_dir(candidate)
         except Exception:
             pass
-        if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True, config=config):
-            if autofix and config.ABXPKG_LIB_DIR != candidate:
-                os.environ["ABXPKG_LIB_DIR"] = str(candidate)
+        if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True):
+            if autofix and config.LIB_DIR != candidate:
+                os.environ["LIB_DIR"] = str(candidate)
             return candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable ABXPKG_LIB_DIR, tried {CANDIDATES}!")
-
-
-def _sql_index_location(config: "ArchiveBoxConfig") -> dict:
-    from archivebox.misc.db import database_display_location, database_exists, is_postgres
-
-    if is_postgres(config):
-        return {
-            "path": database_display_location(),
-            "enabled": True,
-            "is_valid": database_exists(),
-            "is_mount": False,
-        }
-    return {
-        "path": DATABASE_FILE.resolve(),
-        "enabled": True,
-        "is_valid": os.path.isfile(DATABASE_FILE) and os.access(DATABASE_FILE, os.R_OK) and os.access(DATABASE_FILE, os.W_OK),
-        "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
-    }
+        raise OSError(f"ArchiveBox is unable to find a writable LIB_DIR, tried {CANDIDATES}!")
 
 
 def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -324,7 +297,12 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)
                 and os.access(CONSTANTS.CONFIG_FILE, os.W_OK),
             },
-            "SQL_INDEX": _sql_index_location(config),
+            "SQL_INDEX": {
+                "path": DATABASE_FILE.resolve(),
+                "enabled": True,
+                "is_valid": os.path.isfile(DATABASE_FILE) and os.access(DATABASE_FILE, os.R_OK) and os.access(DATABASE_FILE, os.W_OK),
+                "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
+            },
             "ARCHIVE_DIR": {
                 "path": CONSTANTS.ARCHIVE_DIR.resolve(),
                 "enabled": True,
@@ -370,6 +348,11 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 and os.access(tmp_dir, os.W_OK)
                 and tmp_dir_socket_path_is_short_enough(tmp_dir),
             },
+            # "CACHE_DIR": {
+            #     "path": CACHE_DIR.resolve(),
+            #     "enabled": True,
+            #     "is_valid": os.access(CACHE_DIR, os.R_OK) and os.access(CACHE_DIR, os.W_OK),                        # read + write
+            # },
         },
     )
 
@@ -381,9 +364,9 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
 
     config = config or get_config(**config_kwargs)
     try:
-        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.ABXPKG_LIB_DIR
+        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
     except Exception:
-        lib_dir = config.ABXPKG_LIB_DIR
+        lib_dir = config.LIB_DIR
 
     return AttrDict(
         {
@@ -407,7 +390,7 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "enabled": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR),
                 "is_valid": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR) and os.access(CONSTANTS.USER_PLUGINS_DIR, os.R_OK),  # read
             },
-            "ABXPKG_LIB_DIR": {
+            "LIB_DIR": {
                 "path": lib_dir.resolve(),
                 "enabled": True,
                 "is_valid": os.path.isdir(lib_dir) and os.access(lib_dir, os.R_OK) and os.access(lib_dir, os.W_OK),  # read + write

archivebox/config/permissions.py

@@ -1,53 +1,19 @@
 __package__ = "archivebox.config"
 
 import os
-import platform
 import pwd
-import socket
-import stat
-import subprocess
 import sys
-from contextlib import contextmanager, suppress
-from pathlib import Path
+import socket
+import platform
 from typing import cast
 
 from rich import print
 
+from pathlib import Path
+from contextlib import contextmanager
+
 #############################################################################################
 
-
-def is_root_identity(running_uid: int, effective_uid: int) -> bool:
-    return running_uid == 0 or effective_uid == 0
-
-
-def select_archivebox_user(
-    *,
-    running_uid: int,
-    running_gid: int,
-    effective_uid: int,
-    effective_gid: int,
-    sudo_uid: int,
-    sudo_gid: int,
-    data_dir_uid: int,
-    data_dir_gid: int,
-    account_uid: int | None,
-    account_gid: int | None,
-    data_dir_owner_gid: int | None = None,
-    data_dir_owner_exists: bool = True,
-) -> tuple[int, int]:
-    if running_uid == 0:
-        if data_dir_uid != 0 and data_dir_owner_exists:
-            owner_gid = data_dir_owner_gid if data_dir_gid == 0 and data_dir_owner_gid is not None else data_dir_gid
-            return data_dir_uid, owner_gid
-        if account_uid is not None and account_gid is not None:
-            return account_uid, account_gid
-
-    return (
-        effective_uid or running_uid or sudo_uid,
-        effective_gid or running_gid or sudo_gid,
-    )
-
-
 DATA_DIR = Path(os.getcwd())
 
 try:
@@ -58,100 +24,44 @@ except PermissionError:
     DATA_DIR_UID = 0
     DATA_DIR_GID = 0
 
-try:
-    DATA_DIR_OWNER_GID = pwd.getpwuid(DATA_DIR_UID).pw_gid
-    DATA_DIR_OWNER_EXISTS = True
-except KeyError:
-    DATA_DIR_OWNER_GID = None
-    DATA_DIR_OWNER_EXISTS = False
-
 DEFAULT_UID = 911
 DEFAULT_GID = 911
 RUNNING_AS_UID = os.getuid()
 RUNNING_AS_GID = os.getgid()
 EUID = os.geteuid()
 EGID = os.getegid()
-SUDO_UID = int(os.environ.get("SUDO_UID", "0"))
-SUDO_GID = int(os.environ.get("SUDO_GID", "0"))
+SUDO_UID = int(os.environ.get("SUDO_UID", 0))
+SUDO_GID = int(os.environ.get("SUDO_GID", 0))
 USER: str = Path("~").expanduser().resolve().name
 HOSTNAME: str = cast(str, max([socket.gethostname(), platform.node()], key=len))
 
-IS_ROOT = is_root_identity(RUNNING_AS_UID, EUID)
-IN_DOCKER = os.environ.get("IN_DOCKER", "") in ("1", "true", "True", "TRUE", "yes")
+IS_ROOT = RUNNING_AS_UID == 0
+IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
+# IN_DOCKER_COMPOSE =  # TODO: figure out a way to detect if running in docker compose
+
 
 FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
 FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
+try:
+    ARCHIVEBOX_ACCOUNT = pwd.getpwnam("archivebox")
+except KeyError:
+    ARCHIVEBOX_ACCOUNT = None
 
-
-def get_or_create_archivebox_account():
-    try:
-        account = pwd.getpwnam("archivebox")
-    except KeyError:
-        account = None
-
-    if account is None and (RUNNING_AS_UID != 0 or platform.system() != "Linux"):
-        return None
-
-    if account is None:
-        useradd = "/usr/sbin/useradd" if Path("/usr/sbin/useradd").exists() else "useradd"
-        command = [
-            useradd,
-            "--system",
-            "--create-home",
-            "--home-dir",
-            "/var/lib/archivebox",
-            "--shell",
-            "/bin/bash",
-            "archivebox",
-        ]
-        try:
-            subprocess.run(command, check=True)
-        except (OSError, subprocess.CalledProcessError) as err:
-            # Another concurrently starting process may have created it first.
-            try:
-                account = pwd.getpwnam("archivebox")
-            except KeyError:
-                raise RuntimeError(f"Failed to create the archivebox system user with: {' '.join(command)}") from err
-        else:
-            account = pwd.getpwnam("archivebox")
-
-    if RUNNING_AS_UID == 0 and platform.system() == "Linux":
-        # Package removal may preserve the account but remove its home. Repair
-        # only this directory entry; never recursively chown existing data.
-        account_home = Path(account.pw_dir)
-        try:
-            account_home.mkdir(parents=True, exist_ok=True)
-            home_stat = account_home.stat()
-            if (home_stat.st_uid, home_stat.st_gid) != (account.pw_uid, account.pw_gid):
-                os.chown(account_home, account.pw_uid, account.pw_gid)
-        except OSError as err:
-            raise RuntimeError(f"Failed to prepare archivebox account home: {account_home}") from err
-
-    return account
-
-
-ARCHIVEBOX_ACCOUNT = get_or_create_archivebox_account()
-
-ARCHIVEBOX_USER, ARCHIVEBOX_GROUP = select_archivebox_user(
-    running_uid=RUNNING_AS_UID,
-    running_gid=RUNNING_AS_GID,
-    effective_uid=EUID,
-    effective_gid=EGID,
-    sudo_uid=SUDO_UID,
-    sudo_gid=SUDO_GID,
-    data_dir_uid=DATA_DIR_UID,
-    data_dir_gid=DATA_DIR_GID,
-    data_dir_owner_gid=DATA_DIR_OWNER_GID,
-    account_uid=ARCHIVEBOX_ACCOUNT.pw_uid if ARCHIVEBOX_ACCOUNT is not None else None,
-    account_gid=ARCHIVEBOX_ACCOUNT.pw_gid if ARCHIVEBOX_ACCOUNT is not None else None,
-    data_dir_owner_exists=DATA_DIR_OWNER_EXISTS,
-)
+if DATA_DIR_UID != 0:
+    ARCHIVEBOX_USER = DATA_DIR_UID
+    ARCHIVEBOX_GROUP = DATA_DIR_GID
+elif RUNNING_AS_UID == 0 and ARCHIVEBOX_ACCOUNT is not None:
+    ARCHIVEBOX_USER = ARCHIVEBOX_ACCOUNT.pw_uid
+    ARCHIVEBOX_GROUP = ARCHIVEBOX_ACCOUNT.pw_gid
+else:
+    ARCHIVEBOX_USER = EUID or RUNNING_AS_UID or FALLBACK_UID
+    ARCHIVEBOX_GROUP = EGID or RUNNING_AS_GID or FALLBACK_GID
 if not USER:
     try:
         # alternative method 1 to get username
         USER = pwd.getpwuid(ARCHIVEBOX_USER).pw_name
-    except (KeyError, OSError):
-        USER = ""
+    except Exception:
+        pass
 
 if not USER:
     try:
@@ -159,172 +69,41 @@ if not USER:
         import getpass
 
         USER = getpass.getuser()
-    except OSError:
-        USER = ""
+    except Exception:
+        pass
 
 if not USER:
     try:
         # alternative method 3 to get username
         USER = os.getlogin() or "archivebox"
-    except OSError:
+    except Exception:
         USER = "archivebox"
 
 ARCHIVEBOX_USER_EXISTS = False
 try:
     pwd.getpwuid(ARCHIVEBOX_USER)
     ARCHIVEBOX_USER_EXISTS = True
-except KeyError:
+except Exception:
     ARCHIVEBOX_USER_EXISTS = False
 
 
-ROOT_HANDOFF_NAMES = (
-    ".archivebox_id",
-    "ArchiveBox.conf",
-    "index.sqlite3",
-    "index.sqlite3-shm",
-    "index.sqlite3-wal",
-    "archive",
-    "cache",
-    "custom_plugins",
-    "custom_templates",
-    "lib",
-    "logs",
-    "personas",
-    "sonic",
-    "sources",
-    "tmp",
-    "users",
-)
-
-
-def root_data_dir_handoff_paths(data_dir: Path, argv: list[str]) -> tuple[Path, ...]:
-    """Return bounded top-level paths safe to hand from root to archivebox."""
-
-    data_dir = data_dir.resolve()
-    if data_dir == Path("/"):
-        return ()
-
-    try:
-        children = tuple(data_dir.iterdir())
-    except (FileNotFoundError, PermissionError):
-        return ()
-
-    is_init = any(arg in ("init", "install", "--init", "--quick-init") for arg in argv[1:])
-    collection_exists = any((data_dir / marker).exists() for marker in (".archivebox_id", "ArchiveBox.conf", "index.sqlite3"))
-    if not collection_exists and not (is_init and not children):
-        return ()
-
-    top_level_paths = tuple(data_dir / name for name in ROOT_HANDOFF_NAMES if (data_dir / name).exists())
-    log_files = tuple((data_dir / "logs").glob("*.log"))
-    return (data_dir, *top_level_paths, *log_files)
-
-
-def root_should_handoff_data_dir(
-    *,
-    is_root: bool,
-    data_dir_uid: int,
-    account_uid: int | None,
-    data_dir_owner_exists: bool = True,
-) -> bool:
-    """Allow bounded handoff only for root or archivebox-owned collection roots."""
-
-    return is_root and account_uid is not None and (not data_dir_owner_exists or data_dir_uid in (0, account_uid))
-
-
-def root_parent_can_grant_group_traversal(*, parent_uid: int, parent_gid: int, parent_mode: int, account_gid: int) -> bool:
-    """Return whether adding archivebox traversal preserves existing parent access."""
-
-    if parent_uid != 0 or parent_mode & stat.S_IXOTH:
-        return False
-    if parent_gid == account_gid:
-        return not bool(parent_mode & stat.S_IXGRP)
-    return not bool(parent_mode & stat.S_IRWXG)
-
-
-def grant_archivebox_group_traversal(path: Path) -> None:
-    """Let the archivebox account traverse private root-owned parents."""
-
-    if not IS_ROOT or ARCHIVEBOX_ACCOUNT is None:
-        return
-
-    for parent in path.resolve().parents:
-        if parent == Path("/"):
-            continue
-        parent_stat = parent.stat(follow_symlinks=False)
-        if root_parent_can_grant_group_traversal(
-            parent_uid=parent_stat.st_uid,
-            parent_gid=parent_stat.st_gid,
-            parent_mode=parent_stat.st_mode,
-            account_gid=ARCHIVEBOX_ACCOUNT.pw_gid,
-        ):
-            os.chown(parent, -1, ARCHIVEBOX_ACCOUNT.pw_gid, follow_symlinks=False)
-            os.chmod(parent, stat.S_IMODE(parent_stat.st_mode) | stat.S_IXGRP, follow_symlinks=False)
-
-
-def handoff_root_owned_data_dir() -> None:
-    account_uid = ARCHIVEBOX_ACCOUNT.pw_uid if ARCHIVEBOX_ACCOUNT is not None else None
-    if not root_should_handoff_data_dir(
-        is_root=IS_ROOT,
-        data_dir_uid=DATA_DIR_UID,
-        account_uid=account_uid,
-        data_dir_owner_exists=DATA_DIR_OWNER_EXISTS,
-    ):
-        return
-
-    handoff_paths = root_data_dir_handoff_paths(DATA_DIR, sys.argv)
-    if not handoff_paths:
-        return
-
-    # Never walk collection contents. This only grants execute-only traversal
-    # on private parents such as /root so the handed-off data remains reachable.
-    grant_archivebox_group_traversal(DATA_DIR)
-
-    for path in handoff_paths:
-        try:
-            os.chown(path, ARCHIVEBOX_ACCOUNT.pw_uid, ARCHIVEBOX_ACCOUNT.pw_gid, follow_symlinks=False)
-        except (FileNotFoundError, PermissionError):
-            pass
-
-
 #############################################################################################
 
 
 def drop_privileges():
     """If running as root, drop privileges to the data dir owner or archivebox user."""
 
-    # Root-owned uv tools commonly live below /root. Keep the installed package
-    # importable after dropping EUID without changing ownership of the tool env.
-    grant_archivebox_group_traversal(Path(__file__))
-    handoff_root_owned_data_dir()
-
     # Always run ArchiveBox as the user that owns the data dir, or as the
     # archivebox service account when the data dir is root-owned.
-    if os.geteuid() == 0 and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
-        pw_record = pwd.getpwuid(ARCHIVEBOX_USER)
-        supplemental_gid = ARCHIVEBOX_ACCOUNT.pw_gid if ARCHIVEBOX_ACCOUNT is not None else ARCHIVEBOX_GROUP
-        os.initgroups(pw_record.pw_name, supplemental_gid)
-        if os.getegid() != ARCHIVEBOX_GROUP:
-            os.setegid(ARCHIVEBOX_GROUP)
-        if os.geteuid() != ARCHIVEBOX_USER:
+    if os.getuid() == 0:
+        if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
             os.seteuid(ARCHIVEBOX_USER)
 
-        # update environment variables so that subprocesses dont try to write to /root
-        os.environ["HOME"] = pw_record.pw_dir
-        os.environ["LOGNAME"] = pw_record.pw_name
-        os.environ["USER"] = pw_record.pw_name
-        os.environ["XDG_CACHE_HOME"] = str(Path(pw_record.pw_dir) / ".cache")
-        os.environ["XDG_CONFIG_HOME"] = str(Path(pw_record.pw_dir) / ".config")
-        os.environ["XDG_DATA_HOME"] = str(Path(pw_record.pw_dir) / ".local" / "share")
-        os.environ.pop("XDG_RUNTIME_DIR", None)
-        os.environ.pop("ABXBUS_MULTIPROCESS_SEMAPHORE_DIR", None)
-
-        semaphore_dir = Path(pw_record.pw_dir) / ".cache" / "abxbus" / "semaphores"
-        os.environ["ABXBUS_MULTIPROCESS_SEMAPHORE_DIR"] = str(semaphore_dir)
-
-        with suppress(ImportError):
-            from abxbus import retry as abxbus_retry
-
-            abxbus_retry.MULTIPROCESS_SEMAPHORE_DIR = semaphore_dir
+            # update environment variables so that subprocesses dont try to write to /root
+            pw_record = pwd.getpwuid(ARCHIVEBOX_USER)
+            os.environ["HOME"] = pw_record.pw_dir
+            os.environ["LOGNAME"] = pw_record.pw_name
+            os.environ["USER"] = pw_record.pw_name
 
     if ARCHIVEBOX_USER == 0 or not ARCHIVEBOX_USER_EXISTS:
         print(

archivebox/config/version.py

@@ -36,32 +36,26 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):
         # building docs, pyproject.toml is not available
         pass
 
+    # raise Exception('Failed to detect installed archivebox version!')
     return "dev"
 
 
 @cache
 def get_COMMIT_HASH() -> str | None:
-    for env_var in ("ARCHIVEBOX_COMMIT_HASH",):
+    for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"):
         env_commit_hash = os.environ.get(env_var, "").strip()
         if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash):
             return env_commit_hash
 
     if IN_DOCKER:
         try:
-            version_text = Path("/VERSION.txt").read_text()
-            matches = re.findall(r"^COMMIT_HASH=([0-9a-fA-F]{40})$", version_text, re.MULTILINE)
-            if matches:
-                return matches[-1]
+            version_txt = Path("/VERSION.txt").read_text()
+            docker_commit_hashes = re.findall(r"COMMIT_HASH=([0-9a-fA-F]{40})", version_txt)
+            if docker_commit_hashes:
+                return docker_commit_hashes[-1]
         except Exception:
             pass
 
-    try:
-        packaged_commit_hash = (PACKAGE_DIR / "COMMIT_SHA").read_text().strip()
-        if re.fullmatch(r"[0-9a-fA-F]{40}", packaged_commit_hash):
-            return packaged_commit_hash
-    except Exception:
-        pass
-
     def _read_git_file(git_dir: Path, ref: str) -> str | None:
         try:
             return git_dir.joinpath(ref).read_text().strip()
@@ -83,13 +77,6 @@ def get_COMMIT_HASH() -> str | None:
         return None
 
     try:
-        try:
-            pyproject_text = (PACKAGE_DIR.parent / "pyproject.toml").read_text()
-        except FileNotFoundError:
-            pyproject_text = ""
-        if not re.search(r'^name = "archivebox"$', pyproject_text, re.MULTILINE):
-            return None
-
         git_dir = PACKAGE_DIR.parent / ".git"
         if git_dir.is_file():
             gitdir_line = git_dir.read_text().strip()
@@ -109,18 +96,75 @@ def get_COMMIT_HASH() -> str | None:
     except Exception:
         pass
 
+    try:
+        return list((PACKAGE_DIR.parent / ".git/refs/heads/").glob("*"))[0].read_text().strip()
+    except Exception:
+        pass
+
     return None
 
 
 @cache
 def get_BUILD_TIME() -> str:
-    for env_var in ("ARCHIVEBOX_BUILD_TIME", "BUILD_TIME"):
-        build_time = os.environ.get(env_var, "").strip()
-        if build_time:
-            return build_time
+    if IN_DOCKER:
+        try:
+            # if we're in the archivebox official docker image, /VERSION.txt will contain the build time
+            docker_build_end_time = Path("/VERSION.txt").read_text().rsplit("BUILD_END_TIME=")[-1].split("\n", 1)[0]
+            return docker_build_end_time
+        except Exception:
+            pass
 
     src_last_modified_unix_timestamp = (PACKAGE_DIR / "README.md").stat().st_mtime
     return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime("%Y-%m-%d %H:%M:%S %s")
 
 
+# def get_versions_available_on_github(config):
+#     """
+#     returns a dictionary containing the ArchiveBox GitHub release info for
+#     the recommended upgrade version and the currently installed version
+#     """
+
+#     # we only want to perform the (relatively expensive) check for new versions
+#     # when its most relevant, e.g. when the user runs a long-running command
+#     subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help'
+#     long_running_commands = ('add', 'schedule', 'update', 'status', 'server')
+#     if subcommand_run_by_user not in long_running_commands:
+#         return None
+
+#     github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases"
+#     response = requests.get(github_releases_api)
+#     if response.status_code != 200:
+#         stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config)
+#         return None
+#     all_releases = response.json()
+
+#     installed_version = parse_version_string(config['VERSION'])
+
+#     # find current version or nearest older version (to link to)
+#     current_version = None
+#     for idx, release in enumerate(all_releases):
+#         release_version = parse_version_string(release['tag_name'])
+#         if release_version <= installed_version:
+#             current_version = release
+#             break
+
+#     current_version = current_version or all_releases[-1]
+
+#     # recommended version is whatever comes after current_version in the release list
+#     # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest)
+#     try:
+#         recommended_version = all_releases[idx+1]
+#     except IndexError:
+#         recommended_version = None
+
+#     return {'recommended_version': recommended_version, 'current_version': current_version}
+
+# def can_upgrade(config):
+#     if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']:
+#         recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name'])
+#         current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name'])
+#         return recommended_version > current_version
+#     return False
+
+
 VERSION: str = detect_installed_version()

archivebox/config/views.py

@@ -1,32 +1,25 @@
 __package__ = "archivebox.config"
 
-import inspect
 import os
+import inspect
 from pathlib import Path
 from typing import Any
 from urllib.parse import quote, urlencode
-
-from admin_data_views.typing import ItemContext, SectionData, TableContext
-from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view
 from django.http import HttpRequest
 from django.utils import timezone
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 
+from admin_data_views.typing import TableContext, ItemContext, SectionData
+from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
+
 from archivebox.config import CONSTANTS
-from archivebox.machine.models import Binary
 from archivebox.misc.util import parse_date
 
+from archivebox.machine.models import Binary
+
 ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
 INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
-LOG_DETAIL_TAIL_BYTES = 100_000
-
-
-def _read_text_tail(path: Path, max_bytes: int = LOG_DETAIL_TAIL_BYTES) -> str:
-    with path.open("rb") as log_file:
-        size = path.stat().st_size
-        log_file.seek(max(size - max_bytes, 0))
-        return log_file.read(max_bytes).decode("utf-8", errors="replace")
 
 
 def is_superuser(request: HttpRequest) -> bool:
@@ -94,7 +87,7 @@ def obj_to_yaml(obj: Any, indent: int = 0) -> str:
             return f" {obj}"
 
     elif isinstance(obj, (int, float, bool)):
-        return f" {obj!s}"
+        return f" {str(obj)}"
 
     elif callable(obj):
         source = (
@@ -105,7 +98,7 @@ def obj_to_yaml(obj: Any, indent: int = 0) -> str:
         return f" {indent_str}  " + source.replace("\n", f"\n{indent_str}  ")
 
     else:
-        return f" {obj!s}"
+        return f" {str(obj)}"
 
 
 def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]:
@@ -150,9 +143,9 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     rows = {
         "Binary Name": [],
-        "Recorded Version": [],
+        "Found Version": [],
         "Provided By": [],
-        "Recorded Abspath": [],
+        "Found Abspath": [],
     }
 
     db_binaries = get_db_binaries_by_name()
@@ -160,10 +153,18 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
+        binary_is_valid = bool(binary and binary.is_valid)
+
         rows["Binary Name"].append(ItemLink(name, key=name))
-        rows["Recorded Version"].append(binary.version or "-")
-        rows["Provided By"].append(binary.binprovider or "-")
-        rows["Recorded Abspath"].append(binary.abspath or "-")
+
+        if binary_is_valid:
+            rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found")
+            rows["Provided By"].append(binary.binprovider or "-")
+            rows["Found Abspath"].append(binary.abspath or "-")
+        else:
+            rows["Found Version"].append("❌ missing")
+            rows["Provided By"].append("-")
+            rows["Found Abspath"].append("-")
 
     return TableContext(
         title="Binaries",
@@ -299,7 +300,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
                 pid_to_process_id[row.pid] = str(row.id)
                 pid_to_process_type[row.pid] = row.process_type
 
-        except (ImportError, RuntimeError, TypeError, ValueError):
+        except Exception:
             pass
 
     def _pid_cell(pid_value: int | None, uptime_str: str = ""):
@@ -340,7 +341,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
         hours, remainder = divmod(seconds, 3600)
         minutes, secs = divmod(remainder, 60)
         supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
-    except (ImportError, RuntimeError, TypeError, ValueError):
+    except Exception:
         try:
             from archivebox.machine.models import Machine, Process
 
@@ -359,7 +360,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
                 hours, remainder = divmod(seconds, 3600)
                 minutes, secs = divmod(remainder, 60)
                 supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
-        except (RuntimeError, TypeError, ValueError):
+        except Exception:
             pass
 
     rows["PID"].append(_pid_cell(supervisor_pid if isinstance(supervisor_pid, int) else None, supervisor_uptime))
@@ -414,7 +415,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
 def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
 
-    from archivebox.workers.supervisord_util import CONFIG_FILE_NAME, get_existing_supervisord_process, get_sock_file, get_worker
+    from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker, get_sock_file, CONFIG_FILE_NAME
 
     SOCK_FILE = get_sock_file()
     CONFIG_FILE = SOCK_FILE.parent / CONFIG_FILE_NAME
@@ -437,12 +438,8 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
 
     if key == "supervisord":
         relevant_config = CONFIG_FILE.read_text()
-        supervisor_log = CONSTANTS.LOGS_DIR / "supervisord.log"
-        with supervisor_log.open("rb") as log_file:
-            startup_logs = log_file.read(LOG_DETAIL_TAIL_BYTES).decode("utf-8", errors="replace")
-        relevant_logs = _read_text_tail(supervisor_log)
-        startup_lines = [line for line in startup_logs.split("\n") if "RPC interface 'supervisor' initialized" in line]
-        start_ts = startup_lines[-1].split(",", 1)[0] if startup_lines else ""
+        relevant_logs = str(supervisor.readLog(0, 10_000_000))
+        start_ts = [line for line in relevant_logs.split("\n") if "RPC interface 'supervisor' initialized" in line][-1].split(",", 1)[0]
         start_dt = parse_date(start_ts)
         uptime = str(timezone.now() - start_dt).split(".")[0] if start_dt else ""
         supervisor_state = supervisor.getState()
@@ -461,7 +458,7 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
         worker_data = get_worker(supervisor, key)
         proc = worker_data if isinstance(worker_data, dict) else {}
         relevant_config = next((config for config in all_config if config.get("name") == key), {})
-        log_result = supervisor.tailProcessStdoutLog(key, -LOG_DETAIL_TAIL_BYTES, LOG_DETAIL_TAIL_BYTES)
+        log_result = supervisor.tailProcessStdoutLog(key, 0, 10_000_000)
         relevant_logs = str(log_result[0] if isinstance(log_result, tuple) else log_result)
 
     section: SectionData = {
@@ -477,7 +474,7 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
             "Logfile": str(proc.get("stdout_logfile") or ""),
             "Uptime": str(str(proc.get("description") or "").split("uptime ", 1)[-1]),
             "Config": obj_to_yaml(relevant_config) if isinstance(relevant_config, dict) else str(relevant_config),
-            "Recent Logs (last 100 KB)": relevant_logs,
+            "Logs": relevant_logs,
         },
         "help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"},
     }
@@ -531,9 +528,9 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
 def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
 
-    log_file = next(logfile for logfile in CONSTANTS.LOGS_DIR.glob("*.log") if key in logfile.name)
+    log_file = [logfile for logfile in CONSTANTS.LOGS_DIR.glob("*.log") if key in logfile.name][0]
 
-    log_text = _read_text_tail(log_file)
+    log_text = log_file.read_text()
     log_stat = log_file.stat()
 
     section: SectionData = {
@@ -543,8 +540,8 @@ def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
             "Path": str(log_file),
             "Size": f"{log_stat.st_size // 1000} kb",
             "Last Updated": format_parsed_datetime(log_stat.st_mtime),
-            "Tail": "\n".join(log_text.split("\n")[-20:]),
-            "Recent Log (last 100 KB)": log_text,
+            "Tail": "\n".join(log_text[-10_000:].split("\n")[-20:]),
+            "Full Log": log_text,
         },
     }