security: lock down /add/ config for anon/non-staff users

Extends the GHSA-3h23-7824-pj8r / CVE-2026-42601 hardening so that
non-staff users cannot set ANY crawl config on /add/, closing the
alternate injection path via typed plugin_config__<plugin>__<KEY>
fields (e.g. YTDLP_ARGS_EXTRA -> yt-dlp --exec RCE).

Non-staff users (anonymous when PUBLIC_ADD_VIEW=True, or authenticated
non-staff) may now only set: URLs, depth, tags, notes, URL
allow/deny filters, and a public persona preset.

- views: gate all crawl resource-limit overrides (CRAWL_MAX_URLS/SIZE/
  TIMEOUT, TIMEOUT, SNAPSHOT_MAX_SIZE, DELETE_AFTER,
  CRAWL_MAX_CONCURRENT_SNAPSHOTS) behind staff; non-staff inherit them
  from persona/server defaults at hook runtime.
- views/forms: cap permissions for non-staff (anonymous -> public;
  authenticated non-staff -> public/unlisted only, never private).
- template: hide the crawl-limit grid (staff-only) and the permissions
  selector (authenticated-only); plugin grid + advanced section already
  staff-gated.
- tests: update public-submission test to assert limits are rejected,
  and add coverage for the permission tiers and limit-override rejection.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.claude/settings.json

@@ -0,0 +1,5 @@
+{
+  "enabledPlugins": {
+    "pyright-lsp@claude-plugins-official": true
+  }
+}

.claude/settings.local.json

@@ -0,0 +1,51 @@
+{
+  "permissions": {
+    "allow": [
+      "Read(**)",
+      "Glob(**)",
+      "Grep(**)",
+      "Bash(python -m archivebox:*)",
+      "Bash(ls:*)",
+      "Bash(xargs:*)",
+      "Bash(python -c:*)",
+      "Bash(printf:*)",
+      "Bash(pkill:*)",
+      "Bash(python3:*)",
+      "Bash(sqlite3:*)",
+      "WebFetch(domain:github.com)",
+      "Bash(uv add:*)",
+      "Bash(mkdir:*)",
+      "Bash(chmod:*)",
+      "Bash(python -m forum_dl:*)",
+      "Bash(archivebox manage migrate:*)",
+      "Bash(cat:*)",
+      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
+      "Bash(forum-dl:*)",
+      "Bash(pip uninstall:*)",
+      "Bash(python:*)",
+      "Bash(source .venv/bin/activate)",
+      "Bash(mv:*)",
+      "Bash(echo:*)",
+      "Bash(grep:*)",
+      "WebFetch(domain:python-statemachine.readthedocs.io)",
+      "Bash(./bin/run_plugin_tests.sh:*)",
+      "Bash(done)",
+      "Bash(coverage erase:*)",
+      "Bash(gh api:*)"
+    ]
+  },
+  "hooks": {
+    "PreToolUse": [
+      {
+        "matcher": "Bash",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
+            "statusMessage": "Checking working directory..."
+          }
+        ]
+      }
+    ]
+  }
+}

.dockerignore

@@ -3,33 +3,18 @@
 /.heartbeat.json
 /package.json
 /package-lock.json
-__pycache__
-**/__pycache__
 *.pyc
-*.pyo
-*.py[cod]
-*$py.class
+__pycache__/
 .mypy_cache/
 .pytest_cache/
-.ruff_cache/
-.uv-cache/
-.cache/
-.cache*/
 .github/
 .pdm-build/
-.pdm-build/**
 .pdm-python
 .eggs/
-*.egg-info/
 .git/
-!.git/
-.git/*
 .vscode/
 !.git/HEAD
-!.git/packed-refs
-!.git/refs/
-!.git/refs/heads/
-!.git/refs/heads/**
+!.git/refs/heads/*
 
 venv/
 .venv/
@@ -46,7 +31,6 @@ chromeprofile/
 chrome_profile/
 lib/
 out/
-tests/out/
 users/
 archive/
 crawls/
@@ -54,8 +38,6 @@ snapshots/
 logs/
 archivebox-docker-smoke*/
 archivebox-compose-smoke*/
-docker-test/
-docker-test*/
 core
 *.core
 
@@ -68,22 +50,11 @@ dist/
 brew_dist/
 deb_dist/
 pip_dist/
-cache/
-caches/
 assets/
 docker/
 website/
 typings/
 
-# The runtime COPY needs ArchiveBox's package and bin/ entrypoint, not the
-# development UI screenshots or test trees. PNGs are already compressed, so
-# including them creates a large image layer that later cleanup cannot remove.
-publicsite/
-old/
-test/
-tests/
-archivebox/tests/
-
 tmp/
 .tmp/
 data/

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@
 - https://github.com/ArchiveBox/ArchiveBox/issues
 - https://github.com/ArchiveBox/ArchiveBox/pulls
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
 
 ### Development Setup
 

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -30,8 +30,8 @@ body:
       description: Please provide the exact steps you took to trigger the issue (including any shell commands run, URLs visited, buttons clicked, etc.).
       render: markdown
       placeholder: |
-        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 5797:5797 archivebox/archivebox` in iTerm2
-        2. Went to the https://127.0.0.1:5797/add/ page in Google Chrome
+        1. Started ArchiveBox by running: `docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox` in iTerm2
+        2. Went to the https://127.0.0.1:8000/add/ page in Google Chrome
         3. Typed 'https://example.com/xyz' into the 'Add URL' input field
         4. Clicked the 'Add+' button
         5. Got a 500 error and saw the errors below in terminal
@@ -69,7 +69,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:
@@ -138,7 +138,7 @@ body:
             archivebox:
                 image: archivebox/archivebox:latest
                 ports:
-                    - 5797:5797
+                    - 8000:8000
                 volumes:
                     - ./data:/data
                 environment:
@@ -191,6 +191,7 @@ body:
         In the meantime please consider:  
           
           - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
           - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
           - 📚 Reading the [Troubleshooting Guide](https://github.com/ArchiveBox/ArchiveBox/wiki)
           - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)

.github/ISSUE_TEMPLATE/2-feature_request.yml

@@ -71,7 +71,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:
@@ -109,6 +109,9 @@ body:
           required: false
         - label: "I've had a lot of difficulty getting ArchiveBox set up"
           required: false
+        - label: "I would pay $10/mo for a hosted version of ArchiveBox if it had this feature"
+          required: false
+
   - type: markdown
     attributes:
       value: |
@@ -120,5 +123,6 @@ body:
         In the meantime please consider:  
           
           - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 📊 [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
           - 🔍 [Searching the Documentation](https://docs.archivebox.io/) for answers to common questions
           - ✨ Testing out a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases) (issues are often already fixed in our latest `BETA` releases)

.github/ISSUE_TEMPLATE/3-documentation_change.yml

@@ -47,5 +47,6 @@ body:
         In the meantime please consider:  
           
           - 💰 [Donating to support ArchiveBox open-source](https://github.com/ArchiveBox/ArchiveBox/wiki/Donations)
+          - 👨‍✈️ [Hiring us for corporate deployments](https://docs.monadical.com/s/archivebox-consulting-services) with professional support, custom feature development, and help with CAPTCHAs/rate-limits
           - 🔍 [Checking out the new ReadTheDocs Documentation](https://docs.archivebox.io/)
           - ✨ Helping us test a newer [`BETA` release](https://github.com/ArchiveBox/ArchiveBox/releases)

.github/ISSUE_TEMPLATE/config.yml

@@ -6,3 +6,6 @@ contact_links:
   - name: 💬 Chat with the dev team & community on Zulip
     url: https://zulip.archivebox.io
     about: "Join us on our Zulip forum to chat with the developers and other users (it's similar to Discord but self-hosted)."
+  - name: 💁‍♂️ Hire us for professional support with fast response times
+    url: https://docs.monadical.com/s/archivebox-consulting-services
+    about: "We provide hosting, development, and support, including on-prem/cloud w/ SSO & storage, CAPTCHA-solving, proxies, etc."

.github/SECURITY.md

@@ -2,15 +2,18 @@
 
 ---
 
-## Security Documentation
+## Security Information
 
-Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
+Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
+
+https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+
+Also see this section of the README about important caveats when running ArchiveBox:
+
+https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
+
+You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
 
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
-- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
-- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
 - https://github.com/ArchiveBox/ArchiveBox#archive-layout
 - https://github.com/ArchiveBox/ArchiveBox#archivebox-development
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
@@ -20,49 +23,12 @@ Please start by reading these links to understand our existing security model, k
 
 ## Reporting a Vulnerability
 
-~We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.~ We no longer offer CVEs or accept private reporting, 95% of the reports we received were low quality AI slop, so now we just ask for normal Github Issues or PRs.
+We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.
 
-You can contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp) if you have a truly urgent security report, but we do not issue public CVEs.
+1. Go to the Security tab on our Github repo: https://github.com/ArchiveBox/ArchiveBox/security
 
----
+2. Click the ["Report a Vulnerability"](https://github.com/ArchiveBox/ArchiveBox/security/advisories/new) button
 
-## CVEs and Report Credits
+3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
 
-> [!IMPORTANT]
-> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**  
-> Reporters **will still receive credit in release notes** however.
-
-> [!TIP]
-> ~Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.~
-
-Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
-
----
-
-## Threat Model
-
-### All admin users are assumed to be trusted
-
-ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
-
-- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
-- non-logged in anonymous guest users
-
-Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
-
-### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
-
-Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
-
-Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
-
-### XSS, CSRF, CORS, CSP
-
-`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
-
-[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
-
-### SSRF / URL Validation
-
-Note archivebox is **designed to be able to archive any URL the server can reach**, including internal intranet URLs, private IPs, and localhost. Users are expected to configure `URL_ALLOWLIST`/`URL_DENYLIST` or proper firewall/ACL-level blocking at the network level if they need to limit what the server should see.
-It does a basic check to try and avoid recursively archiving snapshots hosted by the archivebox server itself, but it's not a strict security check, it's just to prevent wasted CPU/disk from recursively archiving snapshots in a browser that has the ArchiveBox Browser Extension turned on.
+You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).

.github/STAGING.md

@@ -1,42 +0,0 @@
-# Stable releases require live dev acceptance
-
-The order is **dev CI → dev image publication → automatic Cabbage deployment →
-real capture and browser acceptance → stable publication**. Never
-substitute a published PyPI wheel, healthy container, or local build for this
-acceptance. The release workflow blocks *before* a stable upload unless Cabbage
-has successful evidence from `staging-acceptance.yml` matching all tracked
-source and dependency pins. Only ArchiveBox's own version fields may change
-between dev and stable.
-
-The staging workflow observes the existing image watcher; it cannot deploy or
-restart the application. On Cabbage, install `bin/staging-acceptance-host.sh`
-as root-owned mode 0755 `/usr/local/sbin/archivebox-staging-acceptance`. Its hash
-is checked against the candidate. A dedicated SSH key must use
-`restrict,command="/usr/local/sbin/archivebox-staging-acceptance"` in authorized
-keys. It accepts only `inspect <source-sha>` and `capture <source-sha>`; it cannot
-run caller-provided commands or archive arbitrary URLs.
-
-The `staging` GitHub environment is restricted to dev/main branches. It contains
-`STAGING_SSH_KEY` and `STAGING_KNOWN_HOSTS`, with independently verified host keys.
-Only trusted release events use the private ugNAS runner route. Never expose
-these credentials to PR jobs or copy an operator's existing root key.
-
-Acceptance creates one tagged background crawl per source through the normal
-add/runner path, enabling every snapshot extractor, including TLSNotary. It
-checks installed pins, completed outputs and their files, unexpected/missing
-plugin results, host OOM kills, and real rendered snapshot pages. For this
-release, the explicitly accepted TLSNotary failures and exact missing-Claude-auth
-diagnostics are retained in the report's `limitations` list; they do not disable
-those plugins or excuse missing/truncated successful outputs. Remove the
-TLSNotary exception when its follow-up fix ships. Logs, result inventories and
-screenshots are retained as Actions artifacts. Other failures remain
-failures; do not publish a success deployment record to bypass them. No code
-mounts are allowed. The Cabbage job must pass before its record authorizes a
-stable upload. Queued capture progress is polled; failed tests are not retried.
-
-Browser acceptance follows the default WACZ viewer into the captured document,
-matching its replay base and original URL and checking that its body is visible
-and contains substantial text. An attached viewer iframe can still be
-`about:blank`, so its presence alone cannot prove replay works. Extracted display
-titles and the captured document's `<title>` may legitimately differ; neither
-title equality nor output thumbnails substitute for checking the actual page.

.github/actionlint.yaml

@@ -1,3 +0,0 @@
-self-hosted-runner:
-  labels:
-    - ugnas-linux

.github/configs/ci-linux-build.json

@@ -1,81 +0,0 @@
-{
-  "properties": {
-    "CC_BINARY": {
-      "default": "cc"
-    },
-    "LDAPSEARCH_BINARY": {
-      "default": "ldapsearch"
-    },
-    "INITDB_BINARY": {
-      "default": "initdb"
-    },
-    "PG_CTL_BINARY": {
-      "default": "pg_ctl"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{CC_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "build-essential"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{LDAPSEARCH_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapsearch",
-            "-VV"
-          ]
-        },
-        "apt": {
-          "install_args": [
-            "ldap-utils",
-            "python3-dev",
-            "python3-setuptools",
-            "libssl-dev",
-            "libldap2-dev",
-            "libsasl2-dev",
-            "zlib1g-dev",
-            "libatomic1"
-          ],
-          "version": [
-            "ldapsearch",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ],
-  "postgres_binaries": [
-    {
-      "name": "{INITDB_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "postgresql"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{PG_CTL_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "postgresql"
-          ]
-        }
-      }
-    }
-  ]
-}

.github/configs/ci-macos-brew.json

@@ -1,13 +0,0 @@
-{
-  "properties": {
-    "BREW_BINARY": {
-      "default": "brew"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{BREW_BINARY}",
-      "binproviders": "env"
-    }
-  ]
-}

.github/configs/ci-macos-build.json

@@ -1,44 +0,0 @@
-{
-  "properties": {
-    "CLANG_BINARY": {
-      "default": "clang"
-    },
-    "LDAPVC_BINARY": {
-      "default": "ldapvc"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{CLANG_BINARY}",
-      "binproviders": "env,brew",
-      "overrides": {
-        "brew": {
-          "install_args": [
-            "llvm"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{LDAPVC_BINARY}",
-      "binproviders": "env,brew",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        },
-        "brew": {
-          "install_args": [
-            "openldap"
-          ],
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ]
-}

.github/configs/ci-macos-openldap-env.json

@@ -1,21 +0,0 @@
-{
-  "properties": {
-    "LDAPVC_BINARY": {
-      "default": "ldapvc"
-    }
-  },
-  "required_binaries": [
-    {
-      "name": "{LDAPVC_BINARY}",
-      "binproviders": "env",
-      "overrides": {
-        "env": {
-          "version": [
-            "ldapvc",
-            "-VV"
-          ]
-        }
-      }
-    }
-  ]
-}

.github/configs/ci-tooling.json

@@ -1,349 +0,0 @@
-{
-  "properties": {
-    "CURL_BINARY": {
-      "default": "curl"
-    },
-    "BASH_BINARY": {
-      "default": "bash"
-    },
-    "CHMOD_BINARY": {
-      "default": "chmod"
-    },
-    "DOCKER_BINARY": {
-      "default": "docker"
-    },
-    "GIT_BINARY": {
-      "default": "git"
-    },
-    "GREP_BINARY": {
-      "default": "grep"
-    },
-    "GH_BINARY": {
-      "default": "gh"
-    },
-    "EXPORTFS_BINARY": {
-      "default": "exportfs"
-    },
-    "JQ_BINARY": {
-      "default": "jq"
-    },
-    "MKDIR_BINARY": {
-      "default": "mkdir"
-    },
-    "MOUNT_CIFS_BINARY": {
-      "default": "mount.cifs"
-    },
-    "MOUNT_NFS_BINARY": {
-      "default": "mount.nfs"
-    },
-    "NODE_BINARY": {
-      "default": "node"
-    },
-    "PS_BINARY": {
-      "default": "ps"
-    },
-    "PYTHON_BINARY": {
-      "default": "python"
-    },
-    "PV_BINARY": {
-      "default": "pv"
-    },
-    "SSH_BINARY": {
-      "default": "ssh"
-    },
-    "SED_BINARY": {
-      "default": "sed"
-    },
-    "SMBD_BINARY": {
-      "default": "smbd"
-    },
-    "SUDO_BINARY": {
-      "default": "sudo"
-    },
-    "SYSTEMCTL_BINARY": {
-      "default": "systemctl"
-    },
-    "TAR_BINARY": {
-      "default": "tar"
-    },
-    "TREE_BINARY": {
-      "default": "tree"
-    },
-    "TR_BINARY": {
-      "default": "tr"
-    },
-    "RM_BINARY": {
-      "default": "rm"
-    },
-    "UMOUNT_BINARY": {
-      "default": "umount"
-    },
-    "UV_BINARY": {
-      "default": "uv"
-    },
-    "UNAME_BINARY": {
-      "default": "uname"
-    }
-  },
-  "ci_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{JQ_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{UV_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "bsd_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,pyinfra"
-    }
-  ],
-  "freebsd_binaries": [
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env,pyinfra"
-    }
-  ],
-  "docker_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UNAME_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "deploy_binaries": [
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SSH_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "docker_debug_binaries": [
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{PV_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{TAR_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{TREE_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "docker_validation_binaries": [
-    {
-      "name": "{DOCKER_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GREP_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{MKDIR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{RM_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SED_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SSH_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{TR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UNAME_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "mount_validation_binaries": [
-    {
-      "name": "{BASH_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{CHMOD_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{EXPORTFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "nfs-kernel-server"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{MKDIR_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{MOUNT_CIFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "cifs-utils"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{MOUNT_NFS_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "nfs-common"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{SMBD_BINARY}",
-      "binproviders": "env,apt",
-      "overrides": {
-        "apt": {
-          "install_args": [
-            "samba"
-          ]
-        }
-      }
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{SYSTEMCTL_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{UMOUNT_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "process_binaries": [
-    {
-      "name": "{PS_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "docs_binaries": [
-    {
-      "name": "{CURL_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{SUDO_BINARY}",
-      "binproviders": "env"
-    }
-  ],
-  "git_binaries": [
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "github_binaries": [
-    {
-      "name": "{GH_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ],
-  "node_binaries": [
-    {
-      "name": "{NODE_BINARY}",
-      "binproviders": "env,npm",
-      "min_version": "22.23.1",
-      "overrides": {
-        "npm": {
-          "install_args": [
-            "node@22.23.1"
-          ],
-          "postinstall_scripts": true
-        }
-      }
-    }
-  ],
-  "release_binaries": [
-    {
-      "name": "{UV_BINARY}",
-      "binproviders": "env,pip,brew"
-    },
-    {
-      "name": "{PYTHON_BINARY}",
-      "binproviders": "env"
-    },
-    {
-      "name": "{GH_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{GIT_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{JQ_BINARY}",
-      "binproviders": "env,brew,apt"
-    },
-    {
-      "name": "{CURL_BINARY}",
-      "binproviders": "env,brew,apt"
-    }
-  ]
-}

.github/dependabot.yml

@@ -15,7 +15,7 @@ updates:
         patterns:
           - "*"
   - package-ecosystem: "npm"
-    directory: "/etc"
+    directory: "/"
     target-branch: "dev"
     schedule:
       interval: "monthly"

.github/lychee.toml

@@ -1,38 +0,0 @@
-# Documentation destinations only: do not crawl archived URLs, app routes, or image assets.
-# Manual run from the repository root with Lychee 0.24.2:
-# git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > /tmp/archivebox-link-inputs.txt
-# lychee --config .github/lychee.toml --root-dir "$PWD" --files-from /tmp/archivebox-link-inputs.txt
-# Explicit tracked inputs include README symlinks and let errors in one file leave others checkable.
-include_fragments = "anchor-only"
-include_verbatim = true
-max_retries = 0
-max_concurrency = 4
-timeout = 20
-no_progress = true
-hidden = true
-extensions = ["md", "html", "py", "rst", "yml", "yaml", "sh", "toml"]
-exclude_path = ['(^|/)(tests|data|apidocs|node_modules|\.git)(/|$)']
-# Includes override this exclude; this is a documentation-link allowlist.
-exclude = ['.*']
-include = [
- '^https://github\.com/ArchiveBox/ArchiveBox/blob/[^/]+/.*\.md(?:#|$)',
- '^https://raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/',
- '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki(?:/|#|$)|/?#)',
- '^https?://docs\.archivebox\.io(?:/|$)',
- '^file:.*\.md(?:#|$)',
- '^file:.*/docs/[^/.#]+(?:#|$)',
-]
-fallback_extensions = ["md"]
-remap = [
- # Lychee's Markdown slugger drops variation selectors. Check these against rendered HTML.
-
- '^file:.*/README\.md(#%EF%B8%8F-cli-usage)$ https://github.com/ArchiveBox/ArchiveBox${1}',
- '^file:.*/docs/Install\.md(#option-a-docker--docker-compose-setup-%EF%B8%8F)$ https://github.com/ArchiveBox/ArchiveBox/wiki/Install${1}',
- # The wiki accepts this case alias, but raw file paths are case-sensitive.
- '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/Setting-up-Storage$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/Setting-Up-Storage.md',
-
- # Bare missing wiki pages silently redirect to Home; raw pages return an actual 404.
- # Keep fragment URLs on the rendered wiki to catch published heading drift.
- '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/([^#/?]+)$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/${1}.md',
-]
-preprocess = { command = "bin/lychee_preprocess.py" }

.github/pages/README.md

@@ -1,124 +0,0 @@
-# archivebox.io website
-
-This repository owns its complete website. Build and deploy it without checking
-out another repository's website, fetching shared snippets, or calling a shared
-workflow. Common files are ordinary committed copies; update them deliberately.
-
-## Repository-specific files
-
-- Content: `publicsite/index.html`.
-- Site/gallery generation: `bin/generate_ui_screenshot_gallery.py`.
-- Application capture: `bin/collect_ui_screenshots.sh`.
-- Deployment branch: `dev`.
-
-## Common publication contract
-
-1. Build pull requests without deploying them.
-2. Build website pushes immediately, independently of application capture CI.
-3. Restore validated screenshots from trusted default-branch capture jobs.
-   Keep the original capture revision, timestamp, coverage and image checksums.
-4. Retain `site-screenshots` artifacts for 90 days. After expiry, restore the
-   published image set. Never replace a complete gallery with an empty one when
-   restoration fails. Apple device groups are independently validated artifacts;
-   Android's checked-in bootstrap captures retain their local provenance.
-5. Render current content and layout around those captures. Site revision and
-   capture revision are separate facts; reusing captures must not relabel them.
-6. Serialize production restore/build/deploy under one concurrency group and
-   check out the latest default-branch revision when each build starts. Keep the
-   built revision in provenance; later release-bot commits must not block deploys.
-7. Keep platform-specific build, installation, capture, and completeness checks
-   in the application's existing capture workflow. Site-only edits do not need
-   to rerun those jobs.
-
-## Common presentation contract
-
-The base header, Apps menu and icons, footer, brand styles and navigation behavior
-should remain identical copies across the seven product sites. Put this site's
-name, navigation, CTA and extra footer resources outside the common base files.
-Preserve real links, anchor IDs, no-JavaScript navigation, keyboard operation,
-mobile layout, reduced-motion preferences and existing product-specific content.
-Screenshots stay in English. Translation content, when added, lives in separate
-HTML files; no runtime translation service or cross-repository dependency.
-
-## Editing and building
-
-`base/` contains identical independent copies across the seven product sites.
-Common Python scripts share the same behavior; repository formatters may change
-line wrapping without changing their syntax trees. The base owns the shared logo, Apps menu/icons, footer directory, navigation
-behavior and styles. `site.json` and `nav.html` contain this site's name, routes,
-links and CTA. Product content and additional footer resources stay in the native
-HTML/Jinja/Liquid templates listed above. The `ARCHIVEBOX:*` comments are build
-slots; the renderer fills them with static HTML, never browser-side fetches.
-
-The existing builder calls `site.py render` after generating its content. This
-copies `base/chrome.css` and `base/chrome.js` to `site-base/`, assembles the common
-HTML, validates local resources and writes the site revision to root `build.json`.
-Screenshot manifests are left untouched. No build reads another repository.
-
-Common changes are manual copy/paste edits. Compare `base/` and `site.py` with
-another product repository when updating them; do not add a synchronization job,
-package dependency or reusable cross-repository workflow. `nav.html`, `site.json`
-and the marked workflow build section are intentionally repository-specific.
-
-Build: `uv run --no-project python .github/pages/site.py render _site --source publicsite` after restoring screenshots and refreshing the gallery.
-
-Check the built site with real Chromium (also run by Pages CI):
-
-```sh
-export ABXPKG_LIB_DIR=/tmp/archivebox-site-browser
-CHROME_SCHEMA="$(uv run python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
-uv run abxpkg env --install --deps-from="$CHROME_SCHEMA:required_binaries"
-export CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
-uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py SITE_OUTPUT --evidence /tmp/site-evidence
-```
-
-The browser comes from the normal abxpkg provider/cache path; Playwright drives
-that resolved executable instead of installing a second Chromium build.
-Use this workflow's `SITE_OUTPUT` directory. The check covers desktop/mobile
-layout, keyboard dismissal, no-JavaScript links, local resources and visible
-images, and uploads its screenshots as `site-verification` for 14 days.
-
-## Branding and future translations
-
-`base/chrome.css` owns the shared system font stacks, plum accent and CTA shape,
-including the existing `.button` and `.ab-btn` classes. Product styles own their
-layouts, feature sections and platform-specific button states. Shared system
-fonts need no external font downloads. `site.py` emits the same complete SEO/OG/
-Twitter tags everywhere while retaining each native template's title, description,
-canonical URL, locale and social artwork; edit those values with that page's copy.
-
-For translations, keep complete separate HTML files under `es/`, `fr/` and `zh/`,
-and list their output paths in `site.json` alongside the English pages. Share CSS,
-JS, social artwork and the English screenshot gallery. Translate each page's title
-and description, set its own canonical and `lang` (`zh` for Chinese), and add
-reciprocal `hreflang` links only for translations that actually exist. The renderer
-preserves these tags and computes shared asset paths for nested pages. Localized
-header/footer copy and the small language selector can be added with those pages;
-there is no language redirect, placeholder translation or translation runtime now.
-
-## Screenshot strips
-
-`marquee.py` and `base/marquee.css` / `base/marquee.js` are also identical copies.
-Product homepages choose their placement with an `ARCHIVEBOX:MARQUEE` comment.
-The renderer takes real images, captions, dimensions and links from the generated
-`screenshots/index.html`; there is no second screenshot list or browser fetch.
-Desktop/tablet/mobile galleries contribute their desktop views. Apple uses
-`MARQUEE-CLIENT` and `MARQUEE-SERVER` slots to keep its two products separate.
-Catalog sites without a slot do not load the strip assets.
-
-The strip scrolls once, pauses on hover, focus or manual interaction, and has an
-explicit Play/Pause button. Reduced-motion users start paused. Without JavaScript,
-normal image links and horizontal scrolling remain available. Screenshot capture
-manifests and provenance are never rewritten by this presentation step.
-
-Standalone one- or two-image illustrations remain in their authored source.
-Gallery strips with three or more images use the latest restored captures.
-
-## Keeping the common layout in sync
-
-The comments in `base/header.html`, `base/footer-start.html`,
-`base/footer-end.html`, `base/chrome.css`, and `base/chrome.js` list the peer
-repositories. Update those committed copies together when shared branding,
-Apps links, keyboard behavior, or footer resources change. Preserve each site's
-local navigation and content. The former monorepo `site-chrome` generator is
-retired; builds must use only this repository's files.

.github/pages/artifacts.py

@@ -1,102 +0,0 @@
-"""Common screenshot artifact transport; coverage checks belong to each app."""
-
-import json
-import subprocess
-from pathlib import PurePosixPath
-from urllib.request import urlopen
-
-
-def api(repo, path):
-    return json.loads(subprocess.check_output(["gh", "api", f"repos/{repo}/{path}"]))
-
-
-def trusted(run, repo, workflow, branch, successful=True):
-    return (
-        (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
-        and run["head_branch"] == branch
-        and run["event"] in ("push", "workflow_dispatch", "schedule")
-        and run["path"] == f".github/workflows/{workflow}"
-        and (not successful or run["conclusion"] == "success")
-    )
-
-
-def runs(repo, workflow, branch, successful=True, artifact_names=("site-screenshots",)):
-    # Ask for capture artifacts directly instead of checking every historical
-    # workflow run (most app runs never produce a complete screenshot set).
-    # Artifact creation time also includes fresh captures from old rerun jobs.
-    candidates = []
-    for name in artifact_names:
-        rows = subprocess.check_output(
-            [
-                "gh",
-                "api",
-                f"repos/{repo}/actions/artifacts?name={name}&per_page=100",
-                "--paginate",
-                "--jq",
-                ".artifacts[] | select(.expired == false) | @json",
-            ],
-            text=True,
-        )
-        candidates.extend(json.loads(row) for row in rows.splitlines())
-    seen = set()
-    for artifact in sorted(candidates, key=lambda item: item["created_at"], reverse=True):
-        source = artifact.get("workflow_run") or {}
-        run_id = source.get("id")
-        if (
-            artifact["name"] not in artifact_names
-            or not run_id
-            or run_id in seen
-            or source.get("head_branch") != branch
-            or source.get("head_repository_id") != source.get("repository_id")
-        ):
-            continue
-        seen.add(run_id)
-        run = api(repo, f"actions/runs/{run_id}")
-        if trusted(run, repo, workflow, branch, successful):
-            yield run
-
-
-def names(repo, run):
-    rows = subprocess.check_output(
-        [
-            "gh",
-            "api",
-            f"repos/{repo}/actions/runs/{run['id']}/artifacts?per_page=100",
-            "--paginate",
-            "--jq",
-            ".artifacts[] | @json",
-        ],
-        text=True,
-    )
-    return {artifact["name"] for row in rows.splitlines() if not (artifact := json.loads(row))["expired"]}
-
-
-def download(repo, run, name, destination):
-    subprocess.run(
-        [
-            "gh",
-            "run",
-            "download",
-            str(run["id"]),
-            "--repo",
-            repo,
-            "--name",
-            name,
-            "--dir",
-            str(destination),
-        ],
-        check=True,
-    )
-
-
-def relative_path(name):
-    path = PurePosixPath(name)
-    if not name or path.is_absolute() or ".." in path.parts or any(c in name for c in ":\\?#%"):
-        raise ValueError(f"Unsafe screenshot path: {name}")
-    return path
-
-
-def fetch(base, name):
-    relative_path(name)
-    with urlopen(base + name, timeout=60) as response:
-        return response.read()

.github/pages/base/chrome.css

@@ -1,156 +0,0 @@
-/*
-Shared ArchiveBox website navigation styles and behavior.
-Keep this file in sync with .github/pages/base/chrome.css in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
-*/
-/* Common brand tokens. Native templates own product layout and content. */
-:root {
-  --abx-font-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
-  --abx-font-mono: ui-monospace, "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace;
-  --abx-accent: #9b2854;
-  --abx-accent-hover: #782040;
-  --abx-paper: #fcfaf7;
-  --abx-ink: #28252a;
-  --abx-muted: #716c74;
-  --abx-line: #e9e2e3;
-  --font-sans: var(--abx-font-sans);
-  --font-mono: var(--abx-font-mono);
-  --ab-font-sans: var(--abx-font-sans);
-  --ab-font-serif: var(--abx-font-sans);
-  --ab-font-label: var(--abx-font-sans);
-  --ab-font-mono: var(--abx-font-mono);
-  --ab-accent: var(--abx-accent);
-  --ab-accent-hover: var(--abx-accent-hover);
-  --ab-on-accent: #fff;
-}
-body { font-family: var(--abx-font-sans); }
-code, pre, kbd, samp { font-family: var(--abx-font-mono); }
-/* Reuse existing CTA classes; platform selection continues to control colors. */
-.button, .ab-btn {
-  display: inline-flex; align-items: center; justify-content: center; gap: 10px;
-  min-height: 44px; height: auto; max-width: 100%; padding: 10px 20px;
-  border-radius: 24px; font-family: var(--abx-font-sans);
-  font-size: 14px; font-weight: 650; line-height: 1.4;
-  text-align: center; white-space: normal; text-decoration: none;
-}
-.button.primary, .ab-btn--primary { background: var(--abx-accent); border-color: var(--abx-accent); color: #fff; }
-.button.primary:hover, .ab-btn--primary:hover { background: var(--abx-accent-hover); border-color: var(--abx-accent-hover); color: #fff; }
-.button:focus-visible, .ab-btn:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
-.button svg, .button img, .ab-btn svg, .ab-btn img { flex-shrink: 0; }
-@media (prefers-reduced-motion: reduce) { .button, .ab-btn { transition: none; } .button:hover { transform: none; } }
-
-/* Shared ArchiveBox site chrome. Independent copy; edit alongside the other product sites, without a build dependency. */
-/* Native anchor navigation leaves room for the heading and its surrounding content. */
-html { scroll-behavior: smooth; scroll-padding-top: clamp(96px, 25vh, 240px); }
-@media (prefers-reduced-motion: reduce) { html { scroll-behavior: auto; } }
-.abx-header, .abx-footer { box-sizing: border-box; font-family: var(--abx-font-sans); font-size: 14px; line-height: 1.5; letter-spacing: normal; text-align: left; }
-.abx-header *, .abx-footer * { box-sizing: border-box; }
-.abx-header { position: sticky; top: 0; z-index: 100; display: flex; align-items: center; justify-content: space-between; gap: 24px; width: 100%; max-width: none; margin: 0; padding: 15px max(24px, calc((100% - 1280px) / 2)); border: 0; border-bottom: 1px solid #e9e2e3; background: #fcfaf7f5; color: #28252a; backdrop-filter: blur(16px); }
-.abx-header a, .abx-footer a { text-decoration: none; text-underline-offset: .24em; }
-.abx-header .abx-brand, .abx-footer .abx-brand { display: inline-flex; flex: 0 0 auto; align-items: center; gap: 11px; margin: 0; padding: 0; color: inherit; font-size: 19px; font-weight: 750; line-height: 1.2; letter-spacing: -.5px; white-space: nowrap; }
-.abx-brand .abx-logo { display: block; flex: 0 0 auto; width: 32px; height: 32px; margin: 0; }
-.abx-header .abx-subsite { transform: translateY(3px); margin-left: 2px; color: #716c74; font-size: 12px; font-weight: 450; letter-spacing: 0; }
-.abx-header .abx-nav { display: flex; align-items: center; justify-content: flex-end; flex-wrap: wrap; gap: 0; width: auto; max-width: none; margin: 0; padding: 0; font-size: 13px; font-weight: 600; }
-.abx-nav > a, .abx-nav > details { margin: 0 0 0 15px; padding: 0 0 0 15px; border: 0; border-left: 1px solid #ded4d8; border-radius: 0; background: none; }
-.abx-nav > :first-child { margin-left: 0; padding-left: 0; border-left: 0; }
-.abx-header .abx-nav a { display: inline-flex; align-items: center; min-height: 36px; color: #4d424b; white-space: nowrap; }
-.abx-header .abx-nav a:hover { color: #9b2854; }
-.abx-header .abx-nav .abx-cta { margin-left: 18px; padding: 9px 17px; border: 1px solid #9b2854; border-radius: 24px; background: #9b2854; color: #fff; }
-.abx-header .abx-nav .abx-cta:hover { background: #782040; color: #fff; }
-.abx-header .abx-apps { position: relative; }
-.abx-apps > summary { display: flex; align-items: center; gap: 7px; min-height: 36px; list-style: none; cursor: pointer; color: #4d424b; }
-.abx-apps > summary::-webkit-details-marker { display: none; }
-.abx-apps > summary::before { content: '›'; display: inline-block; color: #9b2854; font-size: 22px; line-height: 1; transition: transform .15s; }
-.abx-apps[open] > summary::before { transform: rotate(90deg); }
-.abx-header .abx-app-links { position: absolute; right: 0; top: calc(100% + 12px); width: 440px; max-width: calc(100vw - 24px); max-height: min(70vh, 580px); overflow-y: auto; padding: 8px; border: 1px solid #e9e2e3; border-radius: 16px; background: #fcfaf7; box-shadow: 0 18px 48px #2d152522, 0 3px 8px #2d152508; }
-.abx-header .abx-app-row { display: flex; align-items: center; gap: 8px; min-height: 44px; padding: 4px 6px; border-radius: 10px; transition: background .15s; }
-.abx-header .abx-app-row:hover, .abx-header .abx-app-row:focus-within { background: #f2e8ed; }
-.abx-header .abx-app-links .abx-app-name { flex: 1; min-width: 0; min-height: 34px; padding: 5px 4px; border-radius: 6px; font-size: 13px; line-height: 1.35; white-space: normal; }
-.abx-header .abx-platforms { display: flex; flex: 0 0 auto; align-items: center; gap: 5px; }
-.abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 auto; height: 30px; min-height: 30px; padding: 5px; border: 1px solid #e8dfe3; border-radius: 9px; background: #fffdfb; color: #4d424b; box-shadow: 0 2px 3px #2d152510, inset 0 1px 0 #fff; transition: transform .15s, box-shadow .15s, border-color .15s; }
-.abx-header .abx-app-links .abx-platform { width: 30px; }
-.abx-header .abx-platform svg { display: block; width: 18px; height: 18px; margin: 0; fill: currentColor; }
-.abx-header .abx-app-links .abx-platform-docker { color: #2386ce; }
-.abx-header .abx-app-links .abx-platform-windows, .abx-header .abx-app-links .abx-platform-app-store, .abx-header .abx-app-links .abx-platform-safari { color: #1879c4; }
-.abx-header .abx-app-links .abx-platform-android { color: #29965d; }
-.abx-header .abx-app-links .abx-platform-firefox { color: #e46b24; }
-.abx-header .abx-app-links .abx-platform-brave { color: #e8552b; }
-.abx-header .abx-app-links .abx-platform-edge { color: #168b95; }
-.abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-platform:focus-visible, .abx-header .abx-app-links .abx-tool:hover, .abx-header .abx-app-links .abx-tool:focus-visible { transform: translateY(-2px); border-color: #c99bb0; background: #fff; box-shadow: 0 5px 10px #2d15251c; }
-.abx-header .abx-app-links .abx-tool { padding: 5px 8px; font-size: 11px; white-space: nowrap; }
-.abx-header .abx-tools-row { margin-top: 5px; padding-top: 9px; border-top: 1px solid #e9e2e3; border-radius: 0 0 10px 10px; }
-.abx-header .abx-tools-row .abx-app-name { flex: 0 0 auto; }
-.abx-header .abx-tools-row .abx-platforms { flex: 0 1 auto; flex-wrap: wrap; justify-content: flex-end; margin-left: auto; }
-.abx-header .abx-app-links code { display: inline; vertical-align: baseline; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 1em; font-weight: inherit; line-height: inherit; margin: 0; padding: 0; border: 0; background: transparent; color: inherit; }
-.abx-header .abx-icon-stack { isolation: isolate; gap: 0; padding: 4px 5px; }
-.abx-header .abx-icon-stack .abx-integration { position: relative; z-index: 1; transform: translateY(var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); transition: transform .18s ease, box-shadow .18s ease, border-color .18s ease; }
-.abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -10px; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(2) { --stack-angle: 9deg; --stack-y: -2px; z-index: 2; color: #569a31; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(3) { --stack-angle: -6deg; --stack-y: 2px; z-index: 3; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(4) { --stack-angle: 7deg; --stack-y: -1px; z-index: 4; color: #f00; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(5) { --stack-angle: -10deg; --stack-y: 1px; z-index: 5; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(6) { --stack-angle: 6deg; --stack-y: -2px; z-index: 6; }
-.abx-header .abx-icon-stack .abx-integration:nth-child(7) { --stack-angle: -5deg; z-index: 7; }
-.abx-header .abx-icon-stack .abx-integration:not(:hover, :focus-visible):has(~ .abx-integration:is(:hover, :focus-visible)) { transform: translate(-6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
-.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) ~ .abx-integration:not(:hover, :focus-visible) { transform: translate(6px, var(--stack-y, 0px)) rotate(var(--stack-angle, -8deg)); }
-.abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { z-index: 20; transform: translateY(-7px) rotate(0deg) scale(1.12); border-color: #c99bb0; box-shadow: 0 8px 14px #2d15252b; }
-@media (hover: none) { .abx-header .abx-icon-stack .abx-integration + .abx-integration { margin-left: -6px; } }
-@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration { transition: none; } }
-.abx-footer { position: relative; z-index: 1; display: block; width: 100%; max-width: none; margin: 72px 0 0; padding: 60px max(24px, calc((100% - 1280px) / 2)) 24px; border: 0; border-top: 3px solid #7e284a; background: radial-gradient(ellipse at 12% 0, #54243b55, transparent 48%), linear-gradient(120deg, #201923, #14151b 75%); color: #ddd2dd; color-scheme: dark; }
-.abx-footer .abx-footer-intro { display: flex; align-items: flex-start; justify-content: space-between; gap: 32px; margin: 0 0 42px; }
-.abx-footer .abx-brand { color: #fff4ed; font-size: 24px; }
-.abx-footer .abx-brand .abx-logo { width: 40px; height: 40px; }
-.abx-footer .abx-tagline { max-width: 430px; margin: 12px 0 0; color: #b8acbb; font-size: 14px; line-height: 1.7; }
-.abx-footer .abx-footer-motto { margin: 2px 0 0; color: #d8b39f; font: italic 25px/1.45 Georgia, serif; text-align: right; }
-.abx-footer .abx-footer-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 42px; margin: 0; }
-.abx-footer .abx-footer-column { margin: 0; padding: 20px 0 0; border: 0; border-top: 1px solid #67505e66; background: none; }
-.abx-footer .abx-footer-column h2 { margin: 0 0 19px; padding: 0; border: 0; color: #e4acbb; font: 650 12px/1.4 var(--abx-font-sans); letter-spacing: .12em; text-transform: uppercase; }
-.abx-footer .abx-footer-column ul { display: block; list-style: none; margin: 0; padding: 0; }
-.abx-footer .abx-footer-column li { display: flex; align-items: center; gap: 10px; margin: 0; padding: 0; min-height: 36px; }
-.abx-footer .abx-footer-column li::before { content: none; }
-.abx-footer .abx-footer-column a { color: #ddd2dd; font-size: 13px; line-height: 1.55; }
-.abx-footer .abx-footer-column a:first-child { padding: 6px 0; }
-.abx-footer a:hover { color: #fff; text-decoration: underline; }
-.abx-footer .abx-source { display: inline-flex; align-items: center; justify-content: center; flex: 0 0 28px; width: 28px; height: 28px; border: 1px solid transparent; border-radius: 50%; color: #a58d9f; }
-.abx-footer .abx-source:hover { color: #f3b5cb; border-color: #80516a; background: #ffffff08; }
-.abx-source svg { width: 15px; height: 15px; fill: currentColor; }
-.abx-footer .abx-footer-bottom { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px 24px; margin-top: 32px; padding-top: 21px; border-top: 1px solid #67505e66; color: #a99bab; font-size: 12px; }
-.abx-footer .abx-footer-bottom p { margin: 0; font-size: inherit; color: inherit; }
-.abx-footer .abx-footer-bottom a { color: #cebbc9; }
-.abx-footer .abx-footer-local { margin: 24px 0 0; padding: 16px 0 0; border: 0; border-top: 1px solid #67505e66; border-radius: 0; background: none; color: #baacbc; font-size: 12px; }
-.abx-footer-local > summary { cursor: pointer; color: #d6bfce; }
-.abx-footer-local > div { padding-top: 18px; }
-.abx-footer-local a { color: #e5aabd; }
-.abx-footer-local .brand, .abx-footer-local .footer-intro, .abx-footer-local .footer-grid, .abx-footer-local .footer-meta { color: inherit; }
-.abx-footer-local .footer-card { background: transparent; border-color: #67505e66; }
-.abx-footer-local h2, .abx-footer-local h3, .abx-footer-local strong { color: #e4d8e3; }
-.abx-footer-local .footer-link-copy span, .abx-footer-local .footer-card-subtitle { color: #baacbc; }
-.abx-header a:focus-visible, .abx-header summary:focus-visible, .abx-footer a:focus-visible, .abx-footer summary:focus-visible { outline: 3px solid #d67c9f; outline-offset: 4px; }
-@media (max-width: 1100px) {
-  .abx-header { flex-wrap: wrap; gap: 8px 24px; padding: 12px 20px; }
-  .abx-header .abx-nav { justify-content: flex-start; }
-  .abx-header .abx-app-links { position: fixed; top: auto; left: 20px; right: auto; width: 440px; margin-top: 10px; }
-}
-@media (max-width: 700px) {
-  .abx-header { position: relative; padding: 14px 18px; }
-  .abx-header .abx-brand { flex-wrap: wrap; row-gap: 5px; }
-  .abx-header .abx-nav { width: 100%; gap: 4px 0; font-size: 12px; }
-  .abx-nav > a, .abx-nav > details { margin-left: 10px; padding-left: 10px; }
-  .abx-header .abx-nav .abx-cta { margin-left: 10px; padding: 7px 13px; }
-  .abx-header .abx-app-links { left: 12px; right: 12px; width: auto; max-height: 60vh; }
-  .abx-footer { padding: 40px 22px 22px; margin-top: 48px; }
-  .abx-footer .abx-footer-intro { display: block; margin-bottom: 30px; }
-  .abx-footer .abx-footer-motto { margin-top: 20px; text-align: left; font-size: 22px; }
-  .abx-footer .abx-footer-grid { grid-template-columns: 1fr; gap: 26px; }
-  .abx-footer .abx-footer-column li { min-height: 40px; }
-}
-@media (prefers-reduced-motion: reduce) { .abx-apps > summary::before, .abx-header .abx-app-row, .abx-header .abx-app-links .abx-platform, .abx-header .abx-app-links .abx-tool { transition: none; } .abx-header .abx-app-links .abx-platform:hover, .abx-header .abx-app-links .abx-tool:hover { transform: none; } }
-
-.abx-footer-local > div > a + a { margin-left: 12px; }
-
-@media (prefers-reduced-motion: reduce) { .abx-header .abx-icon-stack .abx-integration:is(:hover, :focus-visible) { transform: none; } }

.github/pages/base/chrome.js

@@ -1,28 +0,0 @@
-/*
-Shared ArchiveBox website navigation styles and behavior.
-Keep this file in sync with .github/pages/base/chrome.js in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
-*/
-(() => {
-  const menu = document.querySelector('.abx-apps');
-  document.addEventListener('keydown', event => {
-    if (event.key === 'Escape' && menu?.open) { menu.open = false; menu.querySelector('summary').focus(); }
-  });
-  document.addEventListener('click', event => { if (menu?.open && !menu.contains(event.target)) menu.open = false; });
-  function revealAnchor() {
-    let target;
-    try { target = document.getElementById(decodeURIComponent(window.location.hash.slice(1))); } catch { return; }
-    if (!target) return;
-    let details = target.closest('details');
-    let opened = false;
-    while (details) { if (!details.open) { details.open = true; opened = true; } details = details.parentElement.closest('details'); }
-    if (opened) window.requestAnimationFrame(() => target.scrollIntoView());
-  }
-  window.addEventListener('hashchange', revealAnchor);
-  revealAnchor();
-})();

.github/pages/base/footer-end.html

@@ -1,12 +0,0 @@
-<!--
-Shared ArchiveBox website footer.
-Keep this file in sync with .github/pages/base/footer-end.html in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
--->
-  <div class="abx-footer-bottom"><p>Built in the open. Supported by our community.</p><p><a href="https://github.com/ArchiveBox">GitHub organization</a> · <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Security &amp; privacy</a> · <a href="https://archivebox.io">ArchiveBox.io ↗</a></p></div>
-</footer>

.github/pages/base/footer-start.html

@@ -1,13 +0,0 @@
-<!--
-Shared ArchiveBox website footer.
-Keep this file in sync with .github/pages/base/footer-start.html in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
--->
-<footer class="abx-footer" aria-label="ArchiveBox ecosystem">
-  <div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
-  <div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated

.github/pages/base/header.html

@@ -1,24 +0,0 @@
-<!--
-Shared ArchiveBox website navbar and Apps menu.
-Keep this file in sync with .github/pages/base/header.html in:
-ArchiveBox/ArchiveBox, abx-plugins, abxpkg, android-archivebox,
-electron-archivebox, ios-archivebox, and archivebox-browser-extension (all repositories are in the ArchiveBox GitHub organization).
-Each site builds its committed copy independently; there is no central generator.
-Keep site-specific navigation, CTAs, and extra footer links outside this base.
-Also review the inline navbar/footer in abx-dl/website/index.html and
- debian-archivebox/website/index.html when shared links or branding change.
--->
-<header class="abx-header">
-  <a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span><span class="abx-subsite">__SITE_NAME__</span></a>
-  <nav class="abx-nav" aria-label="Main navigation">
-    <details class="abx-apps"><summary>Apps</summary><div class="abx-app-links">
-      <div class="abx-app-row"><a class="abx-app-name" href="https://archivebox.io/#install-compose">ArchiveBox Server</a><div class="abx-platforms"><a class="abx-platform abx-platform-docker" href="https://archivebox.io/#install-compose" aria-label="ArchiveBox Server for Docker" title="ArchiveBox Server for Docker"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M349.9 236.3l-66.1 0 0-59.4 66.1 0 0 59.4zm0-204.3l-66.1 0 0 60.7 66.1 0 0-60.7zm78.2 144.8l-66.1 0 0 59.4 66.1 0 0-59.4zM271.8 104.7l-66.1 0 0 60.1 66.1 0 0-60.1zm78.1 0l-66.1 0 0 60.1 66.1 0 0-60.1zm276.8 100c-14.4-9.7-47.6-13.2-73.1-8.4-3.3-24-16.7-44.9-41.1-63.7l-14-9.3-9.3 14c-18.4 27.8-23.4 73.6-3.7 103.8-8.7 4.7-25.8 11.1-48.4 10.7l-434.7 0c-8.7 50.8 5.8 116.8 44 162.1 37.1 43.9 92.7 66.2 165.4 66.2 157.4 0 273.9-72.5 328.4-204.2 21.4 .4 67.6 .1 91.3-45.2 1.5-2.5 6.6-13.2 8.5-17.1l-13.3-8.9zM115.6 176.8l-66 0 0 59.4 66.1 0 0-59.4-.1 0zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm78.1 0l-66.1 0 0 59.4 66.1 0 0-59.4zm-78.1-72.1l-66.1 0 0 60.1 66.1 0 0-60.1z"/></svg></a><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/#archivebox-serverapp" aria-label="ArchiveBox Server for macOS" title="ArchiveBox Server for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://electron.archivebox.io/" aria-label="ArchiveBox Server for Windows" title="ArchiveBox Server for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://archivebox.io/#install-apt" aria-label="ArchiveBox Server for Linux" title="ArchiveBox Server for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-1Line truncated
-      <div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Desktop</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Desktop for macOS" title="ArchiveBox Desktop for macOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-windows" href="https://windows.archivebox.io/" aria-label="ArchiveBox Desktop for Windows" title="ArchiveBox Desktop for Windows"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M0 93.7l183.6-25.3 0 177.4-183.6 0 0-152.1zM0 418.3l183.6 25.3 0-175.2-183.6 0 0 149.9zm203.8 28l244.2 33.7 0-211.6-244.2 0 0 177.9zm0-380.6l0 180.1 244.2 0 0-213.8-244.2 33.7z"/></svg></a><a class="abx-platform abx-platform-linux" href="https://linux.archivebox.io/" aria-label="ArchiveBox Desktop for Linux" title="ArchiveBox Desktop for Linux"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M220.9 123.3c1 .5 1.8 1.7 3 1.7 1.1 0 2.8-.4 2.9-1.5 .2-1.4-1.9-2.3-3.2-2.9-1.7-.7-3.9-1-5.5-.1-.4 .2-.8 .7-.6 1.1 .3 1.3 2.3 1.1 3.4 1.7zM199 125c1.2 0 2-1.2 3-1.7 1.1-.6 3.1-.4 3.5-1.6 .2-.4-.2-.9-.6-1.1-1.6-.9-3.8-.6-5.5 .1-1.3 .6-3.4 1.5-3.2 2.9 .1 1 1.8 1.5 2.8 1.4zM420 403.8c-3.6-4-5.3-11.6-7.2-19.7-1.8-8.1-3.9-16.8-10.5-22.4-1.3-1.1-2.6-2.1-4-2.9-1.3-.8-2.7-1.5-4.1-2 9.2-27.3 5.6-54.5-3.7-79.1-11.4-30.1-31.3-56.4-46.5-74.4-17.1-21.5-33.7-41.9-33.4-72 .5-45.9 5.1-131.2-75.8-131.3-102.4-.2-76.8 103.4-77.9 135.2-1.7 23.4-6.4 41.8-22.5 64.7-18.9 22.5-45.5 58.8-58.1 96.7-6 17.9-8.8 36.1-6.2 53.3-6.5 5.8-11.4 14.7-16.6 20.2-4.2 4.3-10.3 5.9-17 8.3s-14 6-18.5 14.5c-2.1 3.9-2.8 8.1-2.8 12.4 0 3.9 .6 7.9 1.2 11.8 1.2 8.1 2.5 15.7 .8 20.8-5.2 14.4-5.9 24.4-2.2 31.7 3.8 7.3 11.4 10.5 20.1 12.3 17.3 3.6 40.8 2.7 59.3 12.5 19.8 10.4 39.9 14.1 55.9 10.4 11.6-2.6 21.1-9.6 25.9-20.2 12.5-.1 26.3-5.4 48.3-6.6 14.9-1.2 33.6 5.3 55.1 4.1 .6 2.3 1.4 4.6 2.5 6.7l0 .1c8.3 16.7 23.8 24.3 40.3 23 16.6-1.3 34.1-11 48.3-27.9 13.6-16.4 36-23.2 50.9-32.2 7.4-4.5 13.4-10.1 13.9-18.3 .4-8.2-4.4-17.3-15.5-29.7zM223.8 87.3c9.8-22.2 34.2-21.8 44-.4 6.5 14.2 3.6 30.9-4.3 40.4-1.6-.8-5.9-2.6-12.6-4.9 1.1-1.2 3.1-2.7 3.9-4.6 4.8-11.8-.2-27-9.1-27.3-7.3-.5-13.9 10.8-11.8 23-4.1-2-9.4-3.5-13-4.4-1-6.9-.3-14.6 2.9-21.8zM183.1 75.8c10.1 0 20.8 14.2 19.1 33.5-3.5 1-7.1 2.5-10.2 4.6 1.2-8.9-3.3-20.1-9.6-19.6-8.4 .7-9.8 21.2-1.8 28.1 1 .8 1.9-.2-5.9 5.5-15.6-14.6-10.5-52.1 8.4-52.1zm-13.6 60.7c6.2-4.6 13.6-10 14.1-10.5 4.7-4.4 13.5-14.2 27.9-14.2 7.1 0 15.6 2.3 25.9 8.9 6.3 4.1 11.3 4.4 22.6 9.3 8.4 3.5 13.7 9.7 10.5 18.2-2.6 7.1-11 14.4-22.7 18.1-11.1 3.6-19.8 16-38.2 14.9-3.9-.2-7-1-9.6-2.1-8-3.5-12.2-10.4-20-15-8.6-4.8-13.2-10.4-14.7-15.3-1.4-4.9 0-9 4.2-12.3zm3.3 334c-2.7 35.1-43.9 34.4-75.3 18-29.9-15.8-68.6-6.5-76.5-21.9-2.4-4.7-2.4-12.7 2.6-26.4l0-.2c2.4-7.6 .6-16-.6-23.9-1.2-7.8-1.8-15 .9-20 3.5-6.7 8.5-9.1 14.8-11.3 10.3-3.7 11.8-3.4 19.6-9.9 5.5-5.7 9.5-12.9 14.3-18 5.1-5.5 10-8.1 17.7-6.9 8.1 1.2 15.1 6.8 21.9 16l19.6 35.6c9.5 19.9 43.1 48.4 41 68.9zm-1.4-25.9c-4.1-6.6-9.6-13.6-14.4-19.6 7.1 0 14.2-2.2 16.7-8.9 2.3-6.2 0-14.9-7.4-24.9-13.5-18.2-38.3-32.5-38.3-32.5-13.5-8.4-21.1-18.7-24.6-29.9s-3-23.3-.3-35.2c5.2-22.9 18.6-45.2 27.2-59.2 2.3-1.7 .8 3.2-8.7 20.8-8.5 16.1-24.4 53.3-2.6 82.4 .6-20.7 5.5-41.8 13.8-61.5 12-27.4 37.3-74.9 39.3-112.7 1.1 .8 4.6 3.2 6.2 4.1 4.6 2.7 8.1 6.7 12.6 10.3 12.4 10 28.5 9.2 42.4 1.2 6.2-3.5 11.2-7.5 15.9-9 9.9-3.1 17.8-8.6 22.3-15 7.7 30.4 25.7 74.3 37.2 95.7 6.1 11.4 18.3 35.5 23.6 64.6 3.3-.1 7 .4 10.9 1.4 13.8-35.7-11.7-74.2-23.3-84.9-4.7-4.6-4.9-6.6-2.6-6.5 12.6 11.2 29.2 33.7 35.2 59 2.8 11.6 3.3 23.7 .4 35.7 16.4 6.8 35.9 17.9 30.7 34.8-2.2-.1-3.2 0-4Line truncated
-      <div class="abx-app-row"><a class="abx-app-name" href="https://app.archivebox.io/">ArchiveBox Mobile</a><div class="abx-platforms"><a class="abx-platform abx-platform-apple" href="https://app.archivebox.io/" aria-label="ArchiveBox Mobile for iOS" title="ArchiveBox Mobile for iOS"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M319.1 268.7c-.2-36.7 16.4-64.4 50-84.8-18.8-26.9-47.2-41.7-84.7-44.6-35.5-2.8-74.3 20.7-88.5 20.7-15 0-49.4-19.7-76.4-19.7-55.8 .9-115.1 44.5-115.1 133.2 0 26.2 4.8 53.3 14.4 81.2 12.8 36.7 59 126.7 107.2 125.2 25.2-.6 43-17.9 75.8-17.9 31.8 0 48.3 17.9 76.4 17.9 48.6-.7 90.4-82.5 102.6-119.3-65.2-30.7-61.7-90-61.7-91.9zM262.5 104.5c27.3-32.4 24.8-61.9 24-72.5-24.1 1.4-52 16.4-67.9 34.9-17.5 19.8-27.8 44.3-25.6 71.9 26.1 2 49.9-11.4 69.5-34.3z"/></svg></a><a class="abx-platform abx-platform-android" href="https://android.archivebox.io/" aria-label="ArchiveBox Mobile for Android" title="ArchiveBox Mobile for Android"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!-- Font Awesome Free 5.0.13 by Fonticons, Inc. https://fontawesome.com License: https://creativecommons.org/licenses/by/4.0/ --><path d="M89.6 204.5v115.8c0 15.4-12.1 27.7-27.5 27.7-15.3 0-30.1-12.4-30.1-27.7V204.5c0-15.1 14.8-27.5 30.1-27.5 15.1 0 27.5 12.4 27.5 27.5zm10.8 157c0 16.4 13.2 29.6 29.6 29.6h19.9l.3 61.1c0 36.9 55.2 36.6 55.2 0v-61.1h37.2v61.1c0 36.7 55.5 36.8 55.5 0v-61.1h20.2c16.2 0 29.4-13.2 29.4-29.6V182.1H100.4v179.4zm248-189.1H99.3c0-42.8 25.6-80 63.6-99.4l-19.1-35.3c-2.8-4.9 4.3-8 6.7-3.8l19.4 35.6c34.9-15.5 75-14.7 108.3 0L297.5 34c2.5-4.3 9.5-1.1 6.7 3.8L285.1 73c37.7 19.4 63.3 56.6 63.3 99.4zm-170.7-55.5c0-5.7-4.6-10.5-10.5-10.5-5.7 0-10.2 4.8-10.2 10.5s4.6 10.5 10.2 10.5c5.9 0 10.5-4.8 10.5-10.5zm113.4 0c0-5.7-4.6-10.5-10.2-10.5-5.9 0-10.5 4.8-10.5 10.5s4.6 10.5 10.5 10.5c5.6 0 10.2-4.8 10.2-10.5zm94.8 60.1c-15.1 0-27.5 12.1-27.5 27.5v115.8c0 15.4 12.4 27.7 27.5 27.7 15.4 0 30.1-12.4 30.1-27.7V204.5c0-15.4-14.8-27.5-30.1-27.5z"/></svg></a></div></div>
-      <div class="abx-app-row"><a class="abx-app-name" href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><div class="abx-platforms"><a class="abx-platform abx-platform-chrome" href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj" aria-label="ArchiveBox Browser Extension for Chrome" title="ArchiveBox Browser Extension for Chrome"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc. --><path d="M0 256C0 209.4 12.47 165.6 34.27 127.1L144.1 318.3C166 357.5 207.9 384 256 384C270.3 384 283.1 381.7 296.8 377.4L220.5 509.6C95.9 492.3 0 385.3 0 256zM365.1 321.6C377.4 302.4 384 279.1 384 256C384 217.8 367.2 183.5 340.7 160H493.4C505.4 189.6 512 222.1 512 256C512 397.4 397.4 511.1 256 512L365.1 321.6zM477.8 128H256C193.1 128 142.3 172.1 130.5 230.7L54.19 98.47C101 38.53 174 0 256 0C350.8 0 433.5 51.48 477.8 128V128zM168 256C168 207.4 207.4 168 256 168C304.6 168 344 207.4 344 256C344 304.6 304.6 344 256 344C207.4 344 168 304.6 168 256z"/></svg></a><a class="abx-platform abx-platform-safari" href="https://extension.archivebox.io/" aria-label="ArchiveBox Browser Extension for Safari" title="ArchiveBox Browser Extension for Safari"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><!--! Font Awesome Free 7.3.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2026 Fonticons, Inc. --><path fill="currentColor" d="M274.7 274.7L237.3 237.3 166 346 274.7 274.7zM256 8a248 248 0 1 0 0 496 248 248 0 1 0 0-496zM411.8 182.8l14.8-6.1c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.8 1 2.6 1.7 1.3 1.6 1.7 2.6 .6 2 .6 3.1-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.3-2.6 1.7L418 197.6c-2 .8-4.2 .8-6.1 0s-3.5-2.4-4.3-4.3-.8-4.2 0-6.1 2.4-3.5 4.3-4.3zM314.4 94l6.1-14.8c.8-2 2.4-3.5 4.3-4.3s4.2-.8 6.1 0 3.5 2.4 4.3 4.3 .8 4.2 0 6.1l-6.1 14.8c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0c-1-.4-1.9-1-2.6-1.7s-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1zM256 60c2.1 0 4.2 .8 5.7 2.3S264 65.9 264 68l0 16c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7S253.9 60 256 60zM181 74.9c1-.4 2-.6 3.1-.6s2.1 .2 3.1 .6 1.9 1 2.6 1.7 1.3 1.6 1.7 2.6L197.6 94c.4 1 .7 2 .7 3.1s-.2 2.1-.6 3.1-1 1.9-1.7 2.6-1.6 1.4-2.6 1.8-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.8-1.3-1.7-1.7-2.7l-6.1-14.8c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3zm-63.6 42.5c1.5-1.5 3.5-2.3 5.7-2.3s4.2 .8 5.7 2.3L140 128.7c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3l-11.3-11.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6l0 0zM60 256c0-2.1 .8-4.2 2.3-5.7S65.9 248 68 248l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3l-16 0c-2.1 0-4.2-.8-5.7-2.3S60 258.1 60 256zm40.2 73.2l-14.8 6.1c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1 1-1.9 1.7-2.6 1.6-1.3 2.6-1.7L94 314.4c2-.8 4.2-.8 6.1 0s3.5 2.4 4.3 4.3 .8 4.2 0 6.1-2.4 3.5-4.3 4.3zm4.3-136c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1 2.4-3.5 4.3-4.3 4.2-.8 6.1 0l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1l0 0zM197.6 418l-6.1 14.8c-.4 1-1 1.9-1.7 2.6s-1.6 1.3-2.6 1.7-2 .6-3.1 .6-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6-.6-2-.6-3.1 .2-2.1 .6-3.1l6.1-14.8c.4-1 1-1.9 1.7-2.7s1.6-1.4 2.6-1.8 2-.6 3.1-.6 2.1 .2 3.1 .6 1.9 1 2.6 1.8 1.3 1.7 1.7 2.6 .6 2 .6 3.1-.2 2.1-.7 3.1zM264 444c0 2.1-.8 4.2-2.3 5.7s-3.5 2.3-5.7 2.3-4.2-.8-5.7-2.3-2.3-3.5-2.3-5.7l0-16c0-2.1 .8-4.2 2.3-5.7s3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3 2.3 3.5 2.3 5.7l0 16zm67-6.9c-1 .4-2 .6-3.1 .6s-2.1-.2-3.1-.6-1.9-1-2.6-1.7-1.3-1.6-1.7-2.6L314.4 418c-.8-2-.8-4.2 0-6.1s2.4-3.5 4.3-4.3 4.2-.8 6.1 0 3.5 2.4 4.3 4.3l6.1 14.8c.8 2 .8 4.2 0 6.1s-2.4 3.5-4.3 4.3zm63.6-42.5c-1.5 1.5-3.5 2.3-5.7 2.3s-4.2-.8-5.7-2.3L372 383.3c-1.5-1.5-2.3-3.5-2.3-5.6s.8-4.1 2.3-5.6 3.5-2.3 5.7-2.3 4.2 .8 5.7 2.3l11.3 11.3c1.5 1.5 2.3 3.5 2.3 5.6s-.8 4.1-2.3 5.6l0 0zM286.2 286.2L110.3 401.7 225.8 225.8 401.7 110.3 286.2 286.2zM437.1 331c-.8 2-2.4 3.5-4.3 4.3s-4.2 .8-6.1 0l-14.8-6.1c-2-.8-3.5-2.4-4.3-4.3s-.8-4.2 0-6.1c.4-1 1-1.9 1.7-2.6s1.6-1.3 2.6-1.7 2-.6 3.1-.6 2.1 .2 3.1 .6l14.8 6.1c2 .8 3.5 2.4 4.3 4.3s.8 4.2 0 6.1zm6.9-67l-16 0c-2.1 0-4.2-.8-5.7-2.3s-2.3-3.5-2.3-5.7 .8-4.2 2.3-5.7 3.5-2.3 5.7-2.3l16 0c2.1 0 4.2 .8 5.7 2.3s2.3 3.5 2.3 5.7-.8 4.2-2.3 5.7-3.5 2.3-5.7 2.3z"/></svg></a><a class="abx-platform abx-platform-firefox" href="https://addons.mozilla.org/firefox/addon/archivebox-exporter/" aria-label="ArchiveBox Browser Extension for Firefox" title="ArchiveBox Browser Extension for Firefox"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512Line truncated
-      <div class="abx-app-row abx-plugin-row"><a class="abx-app-name" href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><div class="abx-platforms abx-icon-stack"><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Google Drive storage" title="Google Drive storage"><svg aria-hidden="true" focusable="false" xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="m38.7 419.3 22.6 39c4.7 8.2 11.4 14.7 19.4 19.4l80.6-139.6H0c0 9.1 2.3 18.2 7 26.4z" style="fill:#0066da"/><path d="M256 173.9 175.4 34.3c-7.9 4.7-14.7 11.1-19.4 19.4L7 311.7c-4.6 8-7 17.1-7 26.4h161.3z" style="fill:#00ac47"/><path d="M431.4 477.7c7.9-4.7 14.7-11.1 19.4-19.4l9.4-16.1 44.9-77.7c4.7-8.2 7-17.3 7-26.4H350.7l34.3 67.4z" style="fill:#ea4335"/><path d="m256 173.9 80.6-139.6c-7.9-4.7-17-7-26.4-7H201.8c-9.4 0-18.5 2.6-26.4 7z" style="fill:#00832d"/><path d="M350.7 338.1H161.3L80.6 477.7c7.9 4.7 17 7 26.4 7h298c9.4 0 18.5-2.6 26.4-7z" style="fill:#2684fc"/><path d="M430.5 182.7 356 53.7c-4.7-8.2-11.4-14.7-19.4-19.4L256 173.9l94.7 164.2h161c0-9.1-2.3-18.2-7-26.4z" style="fill:#ffba00"/></svg></a><a class="abx-platform abx-integration" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage#amazon-s3--backblaze-b2--google-drive--etc-rclone" aria-label="Amazon S3 storage" title="Amazon S3 storage"><svg aria-hidden="true" focusable="false" fill="#569A31" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M20.913 13.147l.12-.895c.947.576 1.258.922 1.354 1.071-.16.031-.562.046-1.474-.176zm-2.174 7.988a.547.547 0 0 0-.005.073c0 .084-.207.405-1.124.768a10.28 10.28 0 0 1-1.438.432c-1.405.325-3.128.504-4.853.504-4.612 0-7.412-1.184-7.412-1.704a.547.547 0 0 0-.005-.073L1.81 5.602c.135.078.28.154.432.227.042.02.086.038.128.057.134.062.272.122.417.18l.179.069c.154.058.314.114.478.168.043.013.084.029.13.043.207.065.423.127.646.187l.176.044c.175.044.353.087.534.127a23.414 23.414 0 0 0 .843.17l.121.023c.252.045.508.085.768.122.071.011.144.02.216.03.2.027.4.053.604.077l.24.027c.245.026.49.05.74.07l.081.009c.275.022.552.04.83.056l.233.012c.21.01.422.018.633.025a33.088 33.088 0 0 0 2.795-.026l.232-.011c.278-.016.555-.034.83-.056l.08-.008c.25-.02.497-.045.742-.072l.238-.026c.205-.024.408-.05.609-.077.07-.01.141-.019.211-.03.261-.037.519-.078.772-.122l.111-.02c.215-.04.427-.082.634-.125l.212-.047c.186-.041.368-.085.546-.13l.166-.042c.225-.06.444-.122.654-.189.04-.012.077-.026.115-.038a10.6 10.6 0 0 0 .493-.173c.058-.021.114-.044.17-.066.15-.06.293-.12.43-.185.038-.017.079-.034.116-.052.153-.073.3-.15.436-.228l-.976 7.245c-2.488-.78-5.805-2.292-7.311-3a1.09 1.09 0 0 0-1.088-1.085c-.6 0-1.088.489-1.088 1.088 0 .6.488 1.089 1.088 1.089.196 0 .378-.056.537-.148 1.72.812 5.144 2.367 7.715 3.15zm-7.42-20.047c5.677 0 9.676 1.759 9.75 2.736l-.014.113c-.01.033-.031.067-.048.101-.015.028-.026.057-.047.087-.024.033-.058.068-.09.102-.028.03-.051.06-.084.09-.038.035-.087.07-.133.105-.04.03-.074.06-.119.091-.053.036-.116.071-.177.107-.05.03-.095.06-.15.09-.068.036-.147.073-.222.11-.059.028-.114.057-.177.085-.084.038-.177.074-.268.111-.068.027-.13.054-.203.082-.097.036-.205.072-.31.107-.075.026-.148.053-.228.079-.111.035-.233.069-.35.103-.085.024-.165.05-.253.073-.124.034-.258.065-.389.098-.093.022-.181.046-.278.068-.139.032-.287.061-.433.091-.098.02-.191.041-.293.06-.155.03-.32.057-.482.084-.1.018-.198.036-.302.052-.166.026-.342.048-.515.072-.11.014-.213.03-.325.044-.181.023-.372.041-.56.06-.11.012-.218.025-.332.036-.188.016-.386.029-.58.043-.122.009-.24.02-.364.028-.207.012-.422.02-.635.028-.12.005-.234.012-.354.016a35.605 35.605 0 0 1-2.069 0c-.12-.004-.234-.011-.352-.016-.214-.008-.43-.016-.637-.028-.122-.008-.238-.02-.36-.027-.195-.015-.394-.028-.584-.044-.11-.01-.215-.024-.324-.035-.19-.02-.384-.038-.568-.06l-.315-.044c-.176-.024-.355-.046-.525-.073-.1-.015-.192-.033-.29-.05-.167-.028-.335-.055-.494-.086-.096-.018-.183-.038-.276-.056-.151-.032-.305-.062-.45-.095-.09-.02-.173-.043-.26-.064-.138-.034-.277-.067-.407-.102-.082-.022-.157-.046-.235-.069a11.75 11.75 0 0 1-.368-.108c-.075-.024-.141-.049-.213-.073-.11-.037-.223-.075-.325-.113-.067-.025-.125-.051-.188-.077-.096-.038-.195-.076-.282-.115-.06-.027-.11-.054-.166-.08-.08-.039-.162-.077-.233-.116-.052-.028-.094-.055-.142-.084-.063-.038-.13-.075-.185-.113-.043-.029-.075-.058-.113-.086-.048-.037-.098-.073-.139-.11-.032-.029-.054-.057-.08-.087-.033-.035-.069-.07-.093-.104-.02-.03-.031-.058-.046-.086-.018-.035-.039-.068-.049-.102l-.015-.113c.076-.977 4.074-2.736 9.748-2.736zm12.182 12.124c-.118-.628-.84-1.291-2.31-2.128l.963-7.16a.531.531 0 0 0 .005-.073C22.16 1.581 16.447 0 11.32 0 6.194 0 .482 1.581.482 3.851a.58.58 0 0 0 .005.072L2.819 21.25c.071 2.002 5.236 2.75 8.5 2.75 1.805 0 3.615-.188 5.098-.531.598-.138 1.133-.3 1.592-.48 1.18-.467 1.789-1.053 1.813-1.739l.945-7.018c.557.131 1.016.197 1.389.197.54 0 .902-.137 1.134-.413aLine truncated
-      <div class="abx-app-row abx-tools-row"><a class="abx-app-name" href="https://abx-dl.archivebox.io/">Tools</a><div class="abx-platforms"><a class="abx-tool" href="https://abx-dl.archivebox.io/"><span>One-shot <code>abx-dl</code> CLI</span></a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/tree/dev/archivebox/mcp">MCP</a><a class="abx-tool" href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-api-usage">API</a><a class="abx-tool" href="https://web.archivebox.io">Demo</a></div></div>
-    </div></details>
-    __SITE_NAV__
-  </nav>
-</header>

.github/pages/base/marquee.css

@@ -1,11 +0,0 @@
-/* Identical standalone gallery strip across ArchiveBox product sites. */
-/* The root query container measures the usable width, excluding scrollbars. */
-html { container-type: inline-size; }
-.abx-marquee { min-width: 0; width: 100cqw; max-width: none; margin: 24px calc(50% - 50cqw) 40px; padding: 0; overflow: hidden; }
-.abx-marquee .abx-marquee-viewport { width: 100%; overflow-x: auto; scroll-behavior: auto; overscroll-behavior-x: contain; scrollbar-width: thin; padding: 16px 24px 28px; box-sizing: border-box; }
-.abx-marquee .abx-marquee-track { display: flex; align-items: flex-start; gap: 24px; width: max-content; margin-inline: auto; }
-.abx-marquee .abx-marquee-card { display: block; flex: none; width: min-content; overflow: hidden; border-radius: 10px; background: #fff; box-shadow: 0 0 0 1px rgb(0 0 0 / 12%), 0 2px 4px rgb(0 0 0 / 10%), 0 8px 16px rgb(0 0 0 / 16%); text-decoration: none; color: #533344; }
-.abx-marquee .abx-marquee-card img { display: block; height: clamp(240px, 30vw, 400px); width: auto; max-width: none; max-height: none; object-fit: contain; box-shadow: none; border: 0; border-radius: 0; background: transparent; padding: 0; margin: 0; }
-.abx-marquee :focus-visible { outline: 2px solid #9b2854; outline-offset: 3px; }
-.abx-marquee .abx-marquee-viewport:focus-visible { outline-offset: -3px; }
-@media (max-width: 600px) { .abx-marquee .abx-marquee-track { gap: 16px; } .abx-marquee .abx-marquee-card img { height: 240px; } }

.github/pages/base/marquee.js

@@ -1,44 +0,0 @@
-/* Scroll real gallery links once; keep them usable without JavaScript. */
-(() => {
-  document.querySelectorAll('.abx-marquee').forEach(gallery => {
-    const viewport = gallery.querySelector('.abx-marquee-viewport');
-    const reducedMotion = window.matchMedia('(prefers-reduced-motion: reduce)');
-    let paused = reducedMotion.matches;
-    let hovering = false;
-    let focused = false;
-    let visible = false;
-    let frame;
-    let previous;
-    let offset = viewport.scrollLeft;
-    function tick(time) {
-      const distance = viewport.scrollWidth - viewport.clientWidth;
-      if (distance <= 0) return;
-      if (previous !== undefined) {
-        offset = Math.min(offset + Math.min(time - previous, 50) * 0.035, distance);
-        viewport.scrollTo({ left: offset, behavior: 'instant' });
-      }
-      if (offset >= distance - 1) { paused = true; return; }
-      previous = time;
-      frame = window.requestAnimationFrame(tick);
-    }
-    function sync() {
-      window.cancelAnimationFrame(frame);
-      previous = undefined;
-      offset = viewport.scrollLeft;
-      if (!paused && !hovering && !focused && visible && !document.hidden) frame = window.requestAnimationFrame(tick);
-    }
-    viewport.addEventListener('pointerenter', event => { if (event.pointerType === 'mouse') { hovering = true; sync(); } });
-    viewport.addEventListener('pointerleave', () => { hovering = false; sync(); });
-    viewport.addEventListener('focusin', () => { focused = true; sync(); });
-    viewport.addEventListener('focusout', event => { focused = viewport.contains(event.relatedTarget); sync(); });
-    function pauseForInteraction() { paused = true; sync(); }
-    viewport.addEventListener('pointerdown', pauseForInteraction);
-    viewport.addEventListener('wheel', pauseForInteraction, { passive: true });
-    viewport.addEventListener('keydown', pauseForInteraction);
-    reducedMotion.addEventListener('change', () => { paused = reducedMotion.matches; sync(); });
-    document.addEventListener('visibilitychange', sync);
-    new window.IntersectionObserver(([entry]) => { visible = entry.isIntersecting; sync(); }).observe(gallery);
-    new window.ResizeObserver(sync).observe(viewport);
-    viewport.querySelectorAll('img').forEach(image => image.addEventListener('load', sync));
-  });
-})();

.github/pages/marquee.py

@@ -1,96 +0,0 @@
-"""Render homepage screenshot strips from the site's existing gallery HTML."""
-
-import re
-from html import escape
-from html.parser import HTMLParser
-from urllib.parse import urljoin
-
-
-class Gallery(HTMLParser):
-    """Only gallery figures are captures; navigation and decorative images are not."""
-
-    def __init__(self, base):
-        super().__init__()
-        self.base = base
-        self.stack = []
-        self.captures = []
-
-    def handle_starttag(self, tag, attrs):
-        attrs = dict(attrs)
-        if tag == "img" and any(name == "figure" for name, _ in self.stack):
-            context = [item for _, item in self.stack]
-            profile = next(
-                (
-                    item.get("data-profile") or item.get("data-viewport")
-                    for item in reversed(context)
-                    if item.get("data-profile") or item.get("data-viewport")
-                ),
-                None,
-            )
-            if profile not in (None, "desktop"):
-                return
-            source = attrs.get("src", "")
-            if not source or not attrs.get("alt"):
-                return
-            anchor = next((item["id"] for name, item in reversed(self.stack) if name in ("article", "figure") and item.get("id")), "")
-            server = any(item.get("data-platform") == "server" for item in context)
-            self.captures.append(
-                {
-                    "src": urljoin(self.base, source),
-                    "href": self.base + "#" + anchor if anchor else urljoin(self.base, source),
-                    "title": attrs["alt"],
-                    "width": attrs.get("width", ""),
-                    "height": attrs.get("height", ""),
-                    "product": "server" if server else "client",
-                },
-            )
-        if tag not in {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}:
-            self.stack.append((tag, attrs))
-
-    def handle_endtag(self, tag):
-        for index in range(len(self.stack) - 1, -1, -1):
-            if self.stack[index][0] == tag:
-                del self.stack[index:]
-                break
-
-
-def render(output, baseurl="", featured=()):
-    homepage = output / "index.html"
-    text = homepage.read_text()
-    slots = re.findall(r"<!-- ARCHIVEBOX:(MARQUEE(?:-CLIENT|-SERVER)?) -->", text)
-    if not slots:
-        return False
-    base = "/" + baseurl.strip("/") + "/" if baseurl.strip("/") else "/"
-    gallery = Gallery(base + "screenshots/")
-    gallery.feed((output / "screenshots/index.html").read_text())
-    for slot in slots:
-        product = slot.removeprefix("MARQUEE-").lower() if "-" in slot else None
-        captures = [capture for capture in gallery.captures if product is None or capture["product"] == product]
-        priority = {title: index for index, title in enumerate(featured)}
-        captures.sort(key=lambda capture: priority.get(capture["title"], len(priority)))
-        cards = []
-        for capture in captures:
-            dimensions = "".join(f' {key}="{escape(capture[key])}"' for key in ("width", "height") if capture[key])
-            cards.append(
-                f'<a class="abx-marquee-card" href="{escape(capture["href"])}"><img src="{escape(capture["src"])}" alt="{escape(capture["title"])}"{dimensions} loading="lazy" decoding="async"></a>',
-            )
-        viewport_id = slot.lower()
-        content = (
-            f'<section class="abx-marquee" aria-label="App screenshots"><div class="abx-marquee-viewport" id="{viewport_id}" tabindex="0" aria-label="Screenshots. Scroll sideways to explore."><div class="abx-marquee-track">{"".join(cards)}</div></div></section>'
-            if cards
-            else ""
-        )
-        start, end = f"<!-- ARCHIVEBOX:{slot} -->", f"<!-- /ARCHIVEBOX:{slot} -->"
-        replacement = start + "\n" + content + "\n" + end
-        if end in text:
-            text = re.sub(
-                re.escape(start) + r".*?" + re.escape(end),
-                lambda _, replacement=replacement: replacement,
-                text,
-                count=1,
-                flags=re.DOTALL,
-            )
-        else:
-            text = text.replace(start, replacement, 1)
-    homepage.write_text(text)
-    return True

.github/pages/nav.html

@@ -1,4 +0,0 @@
-<!-- Repository-specific navigation and primary CTA. -->
-<a href="__SITE_BASE__#use-cases">Use cases</a><a href="__SITE_BASE__#configuration">Configuration</a><a href="__SITE_BASE__#layout-security">Layout &amp; security</a><a href="https://github.com/ArchiveBox/ArchiveBox/wiki">Docs</a>
-    <a href="https://github.com/ArchiveBox/ArchiveBox">GitHub ↗</a>
-    <a class="abx-cta" href="__SITE_BASE__#install">Get started</a>

.github/pages/site.json

@@ -1,16 +0,0 @@
-{
-  "name": "Web Archive",
-  "url": "https://archivebox.io/",
-  "pages": [
-    "index.html",
-    "screenshots/index.html"
-  ],
-  "marquee_featured": [
-    "Snapshots grid — desktop",
-    "Snapshot View (header collapsed) — desktop",
-    "AI agent — desktop",
-    "Archive results — desktop",
-    "Add URLs — desktop",
-    "Snapshots table — desktop"
-  ]
-}

.github/pages/site.py

@@ -1,215 +0,0 @@
-"""Assemble and verify this repository's static Pages site (standard library only)."""
-
-import argparse
-import json
-import re
-import shutil
-import subprocess
-from datetime import UTC, datetime
-from html import escape, unescape
-from html.parser import HTMLParser
-from pathlib import Path
-from urllib.parse import unquote, urlsplit
-
-from marquee import render as render_marquee
-
-HERE = Path(__file__).resolve().parent
-ROOT = HERE.parents[1]
-
-
-def replace_block(text, name, content):
-    start, end = f"<!-- ARCHIVEBOX:{name} -->", f"<!-- /ARCHIVEBOX:{name} -->"
-    pattern = re.escape(start) + r".*?" + re.escape(end)
-    if re.search(pattern, text, re.DOTALL):
-        return re.sub(
-            pattern,
-            lambda _: f"{start}\n{content}\n{end}",
-            text,
-            count=1,
-            flags=re.DOTALL,
-        )
-    if text.count(start) != 1:
-        raise ValueError(f"Expected one {name} placeholder")
-    return text.replace(start, f"{start}\n{content}\n{end}", 1)
-
-
-def render_metadata(text):
-    """Keep product copy in native templates; render one consistent SEO block."""
-    page = Page()
-    page.feed(text)
-    title = unescape(re.search(r"<title>(.*?)</title>", text, re.DOTALL)[1])
-    meta = page.metadata
-    description = meta.get("description", meta.get("og:description", ""))
-    image = meta.get("og:image", meta.get("twitter:image", ""))
-    if not description or not image or not page.canonical:
-        raise ValueError("Each page needs a description, social image and canonical")
-    values = {
-        "description": description,
-        "robots": "index,follow,max-image-preview:large",
-        "theme-color": "#9b2854",
-        "og:type": "website",
-        "og:site_name": meta.get("og:site_name", "ArchiveBox"),
-        "og:locale": meta.get("og:locale", "en_US"),
-        "og:url": page.canonical,
-        "og:title": title,
-        "og:description": description,
-        "og:image": image,
-        "og:image:type": meta.get("og:image:type", "image/png"),
-        "og:image:width": meta.get("og:image:width", "1200"),
-        "og:image:height": meta.get("og:image:height", "630"),
-        "og:image:alt": meta.get("og:image:alt", title),
-        "twitter:card": "summary_large_image",
-        "twitter:site": "@ArchiveBoxApp",
-        "twitter:title": title,
-        "twitter:description": description,
-        "twitter:image": image,
-        "twitter:image:alt": meta.get("og:image:alt", title),
-    }
-
-    def strip_managed_tag(match):
-        tag = Page()
-        tag.feed(match[0])
-        if tag.canonicals or any(key in values for key in tag.metadata):
-            return ""
-        # System fonts are shared across sites, without a network font dependency.
-        if "fonts.googleapis.com" in match[0] or "fonts.gstatic.com" in match[0]:
-            return ""
-        return match[0]
-
-    text = re.sub(r"<(?:meta|link)\b[^>]*>", strip_managed_tag, text)
-    text = re.sub(r"<title>.*?</title>", "", text, flags=re.DOTALL)
-    tags = [
-        f"<title>{escape(title)}</title>",
-        f'<link rel="canonical" href="{escape(page.canonical, quote=True)}">',
-    ]
-    for key, value in values.items():
-        attr = "property" if key.startswith("og:") else "name"
-        tags.append(f'<meta {attr}="{key}" content="{escape(value, quote=True)}">')
-    if "<!-- ARCHIVEBOX:METADATA -->" not in text:
-        text = text.replace("</head>", "<!-- ARCHIVEBOX:METADATA -->\n</head>", 1)
-    return replace_block(text, "METADATA", "\n".join(tags))
-
-
-def render(output, source=None, baseurl=""):
-    config = json.loads((HERE / "site.json").read_text())
-    output = output.resolve()
-    if source:
-        source = source.resolve()
-        if output == source or output in source.parents or source in output.parents:
-            raise ValueError("Build output must be separate from the source directory")
-        if output == ROOT or output in ROOT.parents:
-            raise ValueError("Build output must not replace a repository")
-        if output.exists():
-            shutil.rmtree(output)
-        shutil.copytree(source, output)
-    revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
-    assets = output / "site-base"
-    assets.mkdir(parents=True, exist_ok=True)
-    has_marquee = render_marquee(output, baseurl, config.get("marquee_featured", []))
-    common_assets = ["chrome.css", "chrome.js"] + (["marquee.css", "marquee.js"] if has_marquee else [])
-    for name in common_assets:
-        shutil.copyfile(HERE / "base" / name, assets / name)
-    for name in config["pages"]:
-        page = output / name
-        text = render_metadata(page.read_text())
-        base = "../" * (len(Path(name).parts) - 1) or "./"
-        header = (HERE / "base/header.html").read_text().replace("__SITE_NAME__", escape(config["name"]))
-        header = header.replace("__SITE_NAV__", (HERE / "nav.html").read_text().strip())
-        text = replace_block(text, "HEADER", header)
-        for kind in ["FOOTER-START", "FOOTER-END"]:
-            text = replace_block(text, kind, (HERE / "base" / f"{kind.lower()}.html").read_text().strip())
-        text = text.replace("__SITE_BASE__", base).replace("__SITE_REVISION__", revision)
-        resources = f'<link rel="stylesheet" href="{base}site-base/chrome.css?v={revision}">\n<script src="{base}site-base/chrome.js?v={revision}" defer></script>'
-        if has_marquee and name == "index.html":
-            resources += f'\n<link rel="stylesheet" href="{base}site-base/marquee.css?v={revision}">\n<script src="{base}site-base/marquee.js?v={revision}" defer></script>'
-        if "<!-- ARCHIVEBOX:ASSETS -->" not in text:
-            text = text.replace("</head>", "<!-- ARCHIVEBOX:ASSETS -->\n</head>", 1)
-        text = replace_block(text, "ASSETS", resources)
-        page.write_text(text)
-    # Site build provenance is independent of all original screenshot manifests.
-    provenance_path = output / "build.json"
-    provenance = json.loads(provenance_path.read_text()) if provenance_path.exists() else {}
-    provenance.update(
-        revision=revision,
-        generatedAt=datetime.now(UTC).isoformat(),
-        site=config["url"],
-    )
-    provenance_path.write_text(json.dumps(provenance, indent=2) + "\n")
-    (output / ".nojekyll").touch()
-    verify(output, baseurl)
-    print(f"Assembled {len(config['pages'])} pages at {output} from {revision}")
-
-
-class Page(HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self.ids = set()
-        self.metadata = {}
-        self.canonical = ""
-        self.resources = []
-        self.headers = self.footers = self.canonicals = 0
-
-    def handle_starttag(self, tag, attrs):
-        attrs = dict(attrs)
-        if tag == "meta":
-            for key in ["name", "property"]:
-                if attrs.get(key):
-                    self.metadata[attrs[key]] = attrs.get("content", "")
-        if tag == "link" and attrs.get("rel") == "canonical":
-            self.canonical = attrs["href"]
-        if attrs.get("id"):
-            self.ids.add(attrs["id"])
-        classes = attrs.get("class", "").split()
-        self.headers += "abx-header" in classes
-        self.footers += "abx-footer" in classes
-        self.canonicals += tag == "link" and attrs.get("rel") == "canonical"
-        if tag in ["img", "script"] and attrs.get("src"):
-            self.resources.append(attrs["src"])
-        if tag == "link" and attrs.get("rel") in [
-            "stylesheet",
-            "icon",
-            "apple-touch-icon",
-        ]:
-            self.resources.append(attrs["href"])
-
-
-def verify(output, baseurl=""):
-    config = json.loads((HERE / "site.json").read_text())
-    output = output.resolve()
-    prefix = "/" + baseurl.strip("/") if baseurl.strip("/") else ""
-    for name in config["pages"]:
-        path = output / name
-        text = path.read_text()
-        page = Page()
-        page.feed(text)
-        if (page.headers, page.footers, page.canonicals) != (1, 1, 1):
-            raise ValueError(f"{name}: expected exactly one shared header, footer and canonical")
-        if "__SITE_" in text:
-            raise ValueError(f"{name}: unresolved site placeholder")
-        for resource in page.resources:
-            url = urlsplit(resource)
-            if url.scheme or url.netloc or not url.path:
-                continue
-            resource_path = unquote(url.path)
-            if prefix and resource_path.startswith(prefix + "/"):
-                resource_path = resource_path[len(prefix) :]
-            local = (output / resource_path.lstrip("/") if resource_path.startswith("/") else path.parent / resource_path).resolve()
-            if not local.is_relative_to(output) or not local.is_file():
-                raise ValueError(f"{name}: missing local resource {resource}")
-        for asset in ["chrome.css", "chrome.js"]:
-            if (output / "site-base" / asset).read_bytes() != (HERE / "base" / asset).read_bytes():
-                raise ValueError(f"{name}: stale common asset {asset}")
-    print(f"Verified shared layout and local resources for {len(config['pages'])} pages")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("command", choices=["render", "verify"])
-    parser.add_argument("output", type=Path)
-    parser.add_argument("--source", type=Path)
-    parser.add_argument("--baseurl", default="")
-    args = parser.parse_args()
-    if args.command == "render":
-        render(args.output, args.source, args.baseurl)
-    else:
-        verify(args.output, args.baseurl)

.github/pages/verify.py

@@ -1,256 +0,0 @@
-"""Check the built site in real Chromium at desktop and phone widths."""
-
-import argparse
-import json
-import os
-import re
-from functools import partial
-from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
-from html.parser import HTMLParser
-from pathlib import Path
-from threading import Thread
-from urllib.parse import unquote, urljoin, urlsplit
-
-from playwright.sync_api import expect, sync_playwright
-
-HERE = Path(__file__).resolve().parent
-
-
-class Handler(SimpleHTTPRequestHandler):
-    def handle(self):
-        # Browser navigation can cancel an in-flight lazy image response.
-        try:
-            super().handle()
-        except (BrokenPipeError, ConnectionResetError):
-            pass
-
-    def log_message(self, *args):
-        pass
-
-
-class ScreenshotReferences(HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self.references = []
-        self.ids = set()
-
-    def handle_starttag(self, tag, attributes):
-        attrs = dict(attributes)
-        if attrs.get("id"):
-            self.ids.add(attrs["id"])
-        for attribute in ("src", "href"):
-            if attrs.get(attribute):
-                self.references.append(attrs[attribute])
-
-
-def verify_screenshot_references(output, canonical):
-    """Check gallery embeds on every static page, including optional translations."""
-    pages = {}
-    for path in output.rglob("*.html"):
-        page = ScreenshotReferences()
-        page.feed(path.read_text())
-        pages[path.resolve()] = page
-    for path, page in pages.items():
-        page_url = urljoin(canonical, path.relative_to(output.resolve()).as_posix())
-        for reference in page.references:
-            url = urlsplit(urljoin(page_url, reference))
-            if url.netloc != urlsplit(canonical).netloc or "/screenshots/" not in url.path:
-                continue
-            target = output / unquote(url.path).lstrip("/")
-            if target.is_dir():
-                target /= "index.html"
-            target = target.resolve()
-            assert target.is_relative_to(output.resolve()) and target.is_file(), f"{path}: missing screenshot target {reference}"
-            assert not re.match(r"^\d+-", target.name), f"{path}: screenshot filename depends on capture order: {reference}"
-            if url.fragment and target in pages:
-                assert unquote(url.fragment) in pages[target].ids, f"{path}: missing screenshot anchor {reference}"
-
-
-def verify(output, evidence):
-    config = json.loads((HERE / "site.json").read_text())
-    verify_screenshot_references(output, config["url"])
-    evidence.mkdir(parents=True, exist_ok=True)
-    server = ThreadingHTTPServer(("127.0.0.1", 0), partial(Handler, directory=str(output.resolve())))
-    thread = Thread(target=server.serve_forever, daemon=True)
-    thread.start()
-    origin = f"http://127.0.0.1:{server.server_port}"
-    try:
-        with sync_playwright() as playwright:
-            browser = playwright.chromium.launch(executable_path=os.environ.get("CHROME_BINARY"), ignore_default_args=["--hide-scrollbars"])
-            for name in config["pages"]:
-                route = name.removesuffix("index.html")
-                page = browser.new_page(reduced_motion="reduce")
-                errors = []
-                missing = []
-                page.on("pageerror", lambda error, errors=errors: errors.append(str(error)))
-                page.on(
-                    "response",
-                    lambda response, missing=missing: (
-                        missing.append(response.url) if response.url.startswith(origin) and response.status >= 400 else None
-                    ),
-                )
-                for width in [1716, 2560, 390]:
-                    page.set_viewport_size({"width": width, "height": 900})
-                    page.goto(f"{origin}/{route}", wait_until="domcontentloaded")
-                    header = page.locator(".abx-header")
-                    expect(header).to_have_count(1)
-                    expect(header).to_be_visible()
-                    expect(header).to_be_in_viewport()
-                    brand = header.locator(".abx-brand")
-                    expect(brand).to_be_visible()
-                    expect(brand).to_contain_text("ArchiveBox")
-                    expect(brand.locator(".abx-logo")).to_be_visible()
-                    expect(brand.locator(".abx-logo")).to_be_in_viewport()
-                    expect(header).to_have_css("display", "flex")
-                    expect(page.locator(".abx-footer")).to_have_count(1)
-                    expect(page.locator(".abx-footer-column")).to_have_count(3)
-                    expect(header.locator(".abx-cta")).to_be_visible()
-                    menu = header.locator(".abx-apps")
-                    menu.locator("summary").focus()
-                    page.keyboard.press("Enter")
-                    expect(menu).to_have_attribute("open", "")
-                    expect(menu.locator("a").first).to_be_visible()
-                    page.keyboard.press("Escape")
-                    expect(menu).not_to_have_attribute("open", "")
-                    menu.locator("summary").click()
-                    box = header.bounding_box()
-                    assert box
-                    page.mouse.click(box["x"] + 2, box["y"] + 5)
-                    expect(menu).not_to_have_attribute("open", "")
-                    assert page.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth"), (
-                        f"{name}: overflow at {width}"
-                    )
-                    if not route and "screenshots/index.html" in config["pages"]:
-                        strips = page.locator(".abx-marquee")
-                        assert strips.count() > 0, "Homepage must use its screenshot gallery"
-                        for strip in strips.all():
-                            strip.scroll_into_view_if_needed()
-                            bounds = strip.bounding_box()
-                            usable_width = page.evaluate("document.documentElement.clientWidth")
-                            assert bounds and abs(bounds["x"]) < 1
-                            assert abs(bounds["width"] - usable_width) < 1
-                            assert not strip.inner_text().strip(), "Screenshot strips should have no visible captions or links"
-                            cards = strip.locator(".abx-marquee-card")
-                            assert cards.count() > 0, "Screenshot strip is empty"
-                            expect(strip.locator("button")).to_have_count(0)
-                            expect(strip.locator("a:not(.abx-marquee-card)")).to_have_count(0)
-                            viewport = strip.locator(".abx-marquee-viewport")
-                            overflowing = viewport.evaluate(
-                                "node => node.scrollWidth > node.clientWidth",
-                            )
-                            if overflowing:
-                                page.mouse.move(0, 0)
-                                page.emulate_media(reduced_motion="no-preference")
-                                initial = viewport.evaluate("node => node.scrollLeft")
-                                page.wait_for_function(
-                                    "([node, initial]) => node.scrollLeft > initial",
-                                    arg=[viewport.element_handle(), initial],
-                                )
-                                viewport.hover()
-                                stopped = viewport.evaluate("node => node.scrollLeft")
-                                viewport.evaluate(
-                                    "node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
-                                )
-                                assert viewport.evaluate("node => node.scrollLeft") == stopped
-                                page.emulate_media(reduced_motion="reduce")
-                                page.mouse.move(0, 0)
-                                stopped = viewport.evaluate("node => node.scrollLeft")
-                                viewport.evaluate(
-                                    "node => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
-                                )
-                                assert viewport.evaluate("node => node.scrollLeft") == stopped
-                                viewport.focus()
-                                page.keyboard.press("ArrowRight")
-                                assert viewport.evaluate("node => node.scrollWidth > node.clientWidth")
-                            else:
-                                centered = viewport.evaluate(
-                                    """node => {
-                                        const view = node.getBoundingClientRect();
-                                        const track = node.querySelector('.abx-marquee-track').getBoundingClientRect();
-                                        return Math.abs((track.left - view.left) - (view.right - track.right)) < 2;
-                                    }""",
-                                )
-                                assert centered, "Short screenshot strips must be centered"
-                            for card in cards.all():
-                                href = card.get_attribute("href")
-                                assert href and "/screenshots/" in href
-                                response = page.request.get(urljoin(page.url, href))
-                                assert response.ok, f"Broken screenshot link: {href}"
-                                anchor = urlsplit(href).fragment
-                                if anchor:
-                                    assert f'id="{anchor}"' in response.text(), f"Missing screenshot anchor: {href}"
-                    # Decode every local content image using the real browser loader,
-                    # including images in offscreen carousel tracks and lazy galleries.
-                    images = page.locator("main img").evaluate_all("nodes => [...new Set(nodes.map(image => image.src))]")
-                    local_images = [url for url in images if url.startswith(origin + "/")]
-                    page.evaluate(
-                        """async urls => {
-                        for (const url of urls) {
-                            const image = new Image();
-                            image.src = url;
-                            await image.decode();
-                            if (!image.naturalWidth) throw new Error(`Broken image: ${url}`);
-                        }
-                    }""",
-                        local_images,
-                    )
-                    page.evaluate("scrollTo({top: 0, behavior: 'instant'})")
-                    page.evaluate(
-                        "() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
-                    )
-                    page.screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}{width}.png"))
-                    page.locator(".abx-footer").screenshot(path=str(evidence / f"{route.replace('/', '-') or 'home-'}footer-{width}.png"))
-                    for link in header.locator(".abx-nav > a").all():
-                        url = urlsplit(link.get_attribute("href") or "")
-                        if url.scheme or url.netloc:
-                            continue
-                        response = page.request.get(urljoin(page.url, url.geturl()))
-                        assert response.ok, f"Broken navigation: {url.geturl()}"
-                        if url.fragment:
-                            body = response.text()
-                            assert f'id="{url.fragment}"' in body or f"id='{url.fragment}'" in body, (
-                                f"Missing navigation anchor: {url.geturl()}"
-                            )
-                    if route == "" and page.locator("#resources").count():
-                        page.goto(f"{origin}/#resources")
-                        expect(page.locator("#resources")).to_be_visible()
-                assert not errors, errors
-                assert not missing, missing
-                page.close()
-                context = browser.new_context(java_script_enabled=False, viewport={"width": 390, "height": 900})
-                plain = context.new_page()
-                plain.goto(f"{origin}/{route}", wait_until="domcontentloaded")
-                expect(plain.locator(".abx-header")).to_be_visible()
-                expect(plain.locator(".abx-header .abx-brand")).to_be_in_viewport()
-                expect(plain.locator(".abx-header .abx-logo")).to_be_visible()
-                plain.locator(".abx-apps summary").click()
-                expect(plain.locator(".abx-app-links a").first).to_be_visible()
-                expect(plain.locator(".abx-footer-column a").first).to_be_visible()
-                if not route and "screenshots/index.html" in config["pages"]:
-                    assert plain.locator(".abx-marquee-card").count() > 0
-                    for strip in plain.locator(".abx-marquee").all():
-                        bounds = strip.bounding_box()
-                        usable_width = plain.evaluate("document.documentElement.clientWidth")
-                        assert bounds and abs(bounds["x"]) < 1
-                        assert abs(bounds["width"] - usable_width) < 1
-                    assert plain.evaluate("document.documentElement.scrollWidth <= document.documentElement.clientWidth")
-                    expect(plain.locator(".abx-marquee button")).to_have_count(0)
-                    expect(plain.locator(".abx-marquee-card").first).to_be_visible()
-                context.close()
-                print(
-                    f"PASS {name}: desktop/mobile, keyboard, no-JS, images and resources",
-                    flush=True,
-                )
-            browser.close()
-    finally:
-        server.shutdown()
-        server.server_close()
-        thread.join()
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("output", type=Path)
-    parser.add_argument("--evidence", type=Path, default=Path("test-results/site"))
-    args = parser.parse_args()
-    verify(args.output, args.evidence)

.github/scripts/discover_test_matrix.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python3
-"""Discover every test once, batching measured short files."""
-
-import json
-import os
-from pathlib import Path
-
-
-def main() -> None:
-    root = Path(__file__).resolve().parents[2]
-    archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
-
-    if not archivebox_tests:
-        raise SystemExit("No ArchiveBox tests discovered")
-
-    # These are observed per-file pytest seconds, including fixture/startup
-    # time. Shared job setup is paid once per batch. Unknown/long files, optional
-    # dependencies and hosted-only requirements keep independent jobs.
-    durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
-    hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
-    requested = os.environ.get("TEST_FILE", "")
-    batches: list[list[str]] = []
-    batch: list[str] = []
-    batch_seconds = 0
-    for path in archivebox_tests:
-        test_path = path.relative_to(root).as_posix()
-        duration = durations.get(test_path, 60)
-        if duration >= 60 or path in hosted or path.stem == "test_auth_ldap" or test_path == requested:
-            batches.append([test_path])
-            continue
-        if batch and (len(batch) == 8 or batch_seconds + duration > 180):
-            batches.append(batch)
-            batch = []
-            batch_seconds = 0
-        batch.append(test_path)
-        batch_seconds += duration
-    if batch:
-        batches.append(batch)
-
-    matrix: list[dict[str, object]] = []
-    for paths in batches:
-        matrix.append(
-            {
-                "name": f"main/{Path(paths[0]).stem}" if len(paths) == 1 else f"batch/{Path(paths[0]).stem}",
-                "paths": paths,
-                "paths_arg": " ".join(paths),
-                "extra": "ldap" if paths == ["archivebox/tests/test_auth_ldap.py"] else "",
-                "count": len(paths),
-                "ugnas": False,
-            },
-        )
-
-    # Ordinary Linux tests can use either pool. Only genuine hosted-only
-    # requirements need a file header; filenames never enter runner policy.
-    # Keep a bounded NAS share so hosted capacity continues working in parallel.
-    # This is queued work, not a worker count: several waves keep the capped
-    # runners occupied while hosted jobs run in parallel.
-    nas_job_limit = int(os.environ.get("UGNAS_CI_MAX_JOBS", "3"))
-    # Keep long jobs early in the hosted queue. Giving the longest jobs to the
-    # CPU-capped NAS instead created a tail after hosted work had finished; use
-    # its extra slots for a bounded share of shorter work without filename rules.
-    matrix.sort(key=lambda item: sum(durations.get(path, 60) for path in item["paths"]), reverse=True)
-    eligible = [item for item in reversed(matrix) if not any(root / path in hosted for path in item["paths"])]
-    for item in eligible[:nas_job_limit]:
-        item["ugnas"] = True
-
-    discovered_paths = [path for entry in matrix for path in entry["paths"]]
-    if len(discovered_paths) != len(set(discovered_paths)):
-        raise SystemExit("Tests were not discovered exactly once")
-    if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
-        raise SystemExit("Discovered test shard coverage does not match test files")
-
-    print(
-        f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs; "
-        f"{sum(item['ugnas'] for item in matrix)} selected for ugNAS, subject to the availability check",
-    )
-    print(json.dumps(matrix, separators=(",", ":")))
-
-
-if __name__ == "__main__":
-    main()

.github/test-durations.json

@@ -1,176 +0,0 @@
-{
-  "source_runs": [
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37190995942",
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37237793977"
-  ],
-  "measurement": "Unbatched files use successful per-file pytest step wall seconds from run 37237793977, including process and fixture startup, on their actual hosted or ugNAS runner. Already-batched files retain prior JUnit measurements plus 9 seconds startup. Queue and shared job setup are excluded; unknown files remain independent.",
-  "seconds": {
-    "archivebox/tests/test_api_discovery_performance.py": 28,
-    "archivebox/tests/test_api_v1_auth_browser_session.py": 17,
-    "archivebox/tests/test_api_v1_auth_check_api_token.py": 15,
-    "archivebox/tests/test_api_v1_auth_get_api_token.py": 16,
-    "archivebox/tests/test_api_v1_cli_add.py": 52,
-    "archivebox/tests/test_api_v1_cli_remove.py": 33,
-    "archivebox/tests/test_api_v1_cli_schedule.py": 29,
-    "archivebox/tests/test_api_v1_cli_search.py": 15,
-    "archivebox/tests/test_api_v1_cli_update.py": 60,
-    "archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 59,
-    "archivebox/tests/test_api_v1_core_any_id.py": 13,
-    "archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 24,
-    "archivebox/tests/test_api_v1_core_archiveresults.py": 19,
-    "archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 106,
-    "archivebox/tests/test_api_v1_core_snapshots.py": 17,
-    "archivebox/tests/test_api_v1_core_snapshots_rss.py": 16,
-    "archivebox/tests/test_api_v1_core_tag_tag_id.py": 15,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 16,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 16,
-    "archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 16,
-    "archivebox/tests/test_api_v1_core_tags.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 16,
-    "archivebox/tests/test_api_v1_core_tags_autocomplete.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_create.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 15,
-    "archivebox/tests/test_api_v1_core_tags_search.py": 17,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 69,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 15,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 15,
-    "archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 16,
-    "archivebox/tests/test_api_v1_crawls_crawls.py": 16,
-    "archivebox/tests/test_api_v1_machine_binaries.py": 16,
-    "archivebox/tests/test_api_v1_machine_binary_binary_id.py": 16,
-    "archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 16,
-    "archivebox/tests/test_api_v1_machine_machine_current.py": 15,
-    "archivebox/tests/test_api_v1_machine_machine_machine_id.py": 16,
-    "archivebox/tests/test_api_v1_machine_machines.py": 15,
-    "archivebox/tests/test_api_v1_personas_personas.py": 15,
-    "archivebox/tests/test_api_v1_personas_sync.py": 16,
-    "archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 27,
-    "archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 19,
-    "archivebox/tests/test_archive_result_service.py": 49,
-    "archivebox/tests/test_auth_ldap.py": 37,
-    "archivebox/tests/test_binary_service.py": 46,
-    "archivebox/tests/test_cli_add_1.py": 75,
-    "archivebox/tests/test_cli_add_2.py": 98,
-    "archivebox/tests/test_cli_add_3.py": 93,
-    "archivebox/tests/test_cli_add_4.py": 94,
-    "archivebox/tests/test_cli_archiveresult.py": 48,
-    "archivebox/tests/test_cli_background_screenshot.py": 49,
-    "archivebox/tests/test_cli_binary.py": 12,
-    "archivebox/tests/test_cli_config.py": 88,
-    "archivebox/tests/test_cli_crawl.py": 81,
-    "archivebox/tests/test_cli_extract.py": 97,
-    "archivebox/tests/test_cli_extract_input.py": 84,
-    "archivebox/tests/test_cli_help.py": 16,
-    "archivebox/tests/test_cli_init.py": 111,
-    "archivebox/tests/test_cli_install.py": 38,
-    "archivebox/tests/test_cli_list.py": 116,
-    "archivebox/tests/test_cli_machine.py": 19,
-    "archivebox/tests/test_cli_manage.py": 32,
-    "archivebox/tests/test_cli_mcp.py": 30,
-    "archivebox/tests/test_cli_oneshot.py": 15,
-    "archivebox/tests/test_cli_persona.py": 56,
-    "archivebox/tests/test_cli_piping.py": 118,
-    "archivebox/tests/test_cli_pluginmap.py": 12,
-    "archivebox/tests/test_cli_process.py": 12,
-    "archivebox/tests/test_cli_remove.py": 46,
-    "archivebox/tests/test_cli_run_1.py": 93,
-    "archivebox/tests/test_cli_run_2.py": 111,
-    "archivebox/tests/test_cli_run_3.py": 51,
-    "archivebox/tests/test_cli_run_4.py": 78,
-    "archivebox/tests/test_cli_schedule.py": 59,
-    "archivebox/tests/test_cli_search.py": 25,
-    "archivebox/tests/test_cli_server.py": 126,
-    "archivebox/tests/test_cli_shell.py": 17,
-    "archivebox/tests/test_cli_snapshot.py": 55,
-    "archivebox/tests/test_cli_status.py": 46,
-    "archivebox/tests/test_cli_tag.py": 12,
-    "archivebox/tests/test_cli_update.py": 62,
-    "archivebox/tests/test_cli_update_reindex_snapshots.py": 84,
-    "archivebox/tests/test_cli_version.py": 81,
-    "archivebox/tests/test_config_DELETE_AFTER.py": 145,
-    "archivebox/tests/test_config_MAX_limits.py": 25,
-    "archivebox/tests/test_config_ONLY_NEW.py": 15,
-    "archivebox/tests/test_config_SAVE_TITLE.py": 59,
-    "archivebox/tests/test_config_URL_filters.py": 15,
-    "archivebox/tests/test_core_config.py": 11,
-    "archivebox/tests/test_core_forms.py": 14,
-    "archivebox/tests/test_crawl_history.py": 60,
-    "archivebox/tests/test_crawl_pause.py": 11,
-    "archivebox/tests/test_crawl_runner.py": 22,
-    "archivebox/tests/test_crawl_runner_2.py": 27,
-    "archivebox/tests/test_crawl_service.py": 19,
-    "archivebox/tests/test_frozen_crawl_config.py": 26,
-    "archivebox/tests/test_hook_result_lifecycle.py": 48,
-    "archivebox/tests/test_hooks.py": 54,
-    "archivebox/tests/test_issue_1896.py": 115,
-    "archivebox/tests/test_machine_models.py": 26,
-    "archivebox/tests/test_machine_service.py": 77,
-    "archivebox/tests/test_migrations_04_to_09.py": 25,
-    "archivebox/tests/test_migrations_07_to_09.py": 86,
-    "archivebox/tests/test_migrations_08_to_09.py": 85,
-    "archivebox/tests/test_migrations_fresh.py": 55,
-    "archivebox/tests/test_misc_checks.py": 12,
-    "archivebox/tests/test_misc_monkey_patches.py": 12,
-    "archivebox/tests/test_opencode_agent_1.py": 73,
-    "archivebox/tests/test_opencode_agent_2.py": 141,
-    "archivebox/tests/test_opencode_agent_3.py": 61,
-    "archivebox/tests/test_opencode_browser_1.py": 139,
-    "archivebox/tests/test_opencode_browser_2.py": 164,
-    "archivebox/tests/test_opencode_browser_3.py": 125,
-    "archivebox/tests/test_orm_helpers.py": 14,
-    "archivebox/tests/test_permissions.py": 12,
-    "archivebox/tests/test_persona_casing.py": 45,
-    "archivebox/tests/test_persona_runtime.py": 30,
-    "archivebox/tests/test_postgres_backend.py": 79,
-    "archivebox/tests/test_process_runtime_paths.py": 20,
-    "archivebox/tests/test_process_service.py": 40,
-    "archivebox/tests/test_proof_previews.py": 24,
-    "archivebox/tests/test_recursive_crawl_1.py": 361,
-    "archivebox/tests/test_recursive_crawl_2.py": 55,
-    "archivebox/tests/test_recursive_crawl_3.py": 67,
-    "archivebox/tests/test_release_notes.py": 12,
-    "archivebox/tests/test_release_source_fingerprint.py": 13,
-    "archivebox/tests/test_release_workflow.py": 15,
-    "archivebox/tests/test_resource_admission.py": 13,
-    "archivebox/tests/test_runner_deletion.py": 91,
-    "archivebox/tests/test_runner_watch.py": 75,
-    "archivebox/tests/test_search.py": 142,
-    "archivebox/tests/test_serve_static.py": 19,
-    "archivebox/tests/test_server_automatic_url.py": 22,
-    "archivebox/tests/test_server_security_browser.py": 121,
-    "archivebox/tests/test_settings_signal_webhooks.py": 13,
-    "archivebox/tests/test_setup_script.py": 12,
-    "archivebox/tests/test_shutdown_util.py": 13,
-    "archivebox/tests/test_snapshot_filesystem_migration.py": 16,
-    "archivebox/tests/test_snapshot_output_stacks.py": 63,
-    "archivebox/tests/test_snapshot_ref_lookup.py": 15,
-    "archivebox/tests/test_snapshot_service.py": 48,
-    "archivebox/tests/test_snapshot_url_length.py": 16,
-    "archivebox/tests/test_tag_service.py": 15,
-    "archivebox/tests/test_takeover_util_1.py": 60,
-    "archivebox/tests/test_takeover_util_2.py": 62,
-    "archivebox/tests/test_takeover_util_3.py": 76,
-    "archivebox/tests/test_takeover_util_4.py": 84,
-    "archivebox/tests/test_test_harness.py": 12,
-    "archivebox/tests/test_ui_add_view.py": 26,
-    "archivebox/tests/test_ui_add_view_runtime.py": 120,
-    "archivebox/tests/test_ui_admin_apitoken.py": 16,
-    "archivebox/tests/test_ui_admin_archiveresult.py": 18,
-    "archivebox/tests/test_ui_admin_config_widget.py": 13,
-    "archivebox/tests/test_ui_admin_crawl.py": 25,
-    "archivebox/tests/test_ui_admin_links.py": 23,
-    "archivebox/tests/test_ui_admin_machine.py": 18,
-    "archivebox/tests/test_ui_admin_persona.py": 19,
-    "archivebox/tests/test_ui_admin_snapshot.py": 138,
-    "archivebox/tests/test_ui_admin_tag.py": 16,
-    "archivebox/tests/test_ui_admin_user.py": 16,
-    "archivebox/tests/test_ui_config_views.py": 20,
-    "archivebox/tests/test_ui_live_progress.py": 121,
-    "archivebox/tests/test_ui_public_snapshot.py": 72,
-    "archivebox/tests/test_ui_savepagenow.py": 31,
-    "archivebox/tests/test_urls.py": 105,
-    "archivebox/tests/test_util.py": 13,
-    "archivebox/tests/test_version_metadata.py": 18,
-    "archivebox/tests/test_zip_preview.py": 29
-  }
-}

.github/workflows/ci.yml

@@ -1,103 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-    branches: [dev, main]
-  push:
-    branches: [dev, main]
-  workflow_dispatch:
-
-permissions:
-  actions: read
-  contents: read
-  packages: write
-  security-events: write
-
-concurrency:
-  group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }}
-  cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
-
-jobs:
-  prepare:
-    permissions:
-      contents: write
-    uses: ./.github/workflows/prepare-release.yml
-    secrets: inherit
-
-  lint:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Linters
-    uses: ./.github/workflows/lint.yml
-    secrets: inherit
-
-  install-cli-platform:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Install and CLI platform compatibility
-    uses: ./.github/workflows/test.yml
-    secrets: inherit
-
-  tests:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Discovered test matrix
-    uses: ./.github/workflows/test-parallel.yml
-    secrets: inherit
-
-  codeql:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: CodeQL
-    uses: ./.github/workflows/codeql.yml
-    secrets: inherit
-
-  python-artifacts:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Tested Python artifacts
-    uses: ./.github/workflows/pip.yml
-    secrets: inherit
-
-  docker-digests:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Tested Docker images
-    uses: ./.github/workflows/docker.yml
-    with:
-      push_digests: ${{ github.event_name == 'push' && (github.ref_name == 'dev' || github.ref_name == 'main') }}
-    secrets: inherit
-
-  required:
-    name: All required CI lanes
-    if: ${{ always() && needs.prepare.outputs.run_jobs == 'true' }}
-    needs:
-      - prepare
-      - lint
-      - install-cli-platform
-      - tests
-      - codeql
-      - python-artifacts
-      - docker-digests
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Verify every required lane succeeded
-        env:
-          LINT_RESULT: ${{ needs.lint.result }}
-          INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
-          TESTS_RESULT: ${{ needs.tests.result }}
-          CODEQL_RESULT: ${{ needs.codeql.result }}
-          PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
-          DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
-        run: |
-          for result in \
-            "$LINT_RESULT" \
-            "$INSTALL_CLI_PLATFORM_RESULT" \
-            "$TESTS_RESULT" \
-            "$CODEQL_RESULT" \
-            "$PYTHON_ARTIFACTS_RESULT" \
-            "$DOCKER_DIGESTS_RESULT"
-          do
-            [[ "$result" == success ]]
-          done
-          echo 'All required CI lanes and tested release artifacts succeeded.'

.github/workflows/claude.yml

@@ -17,24 +17,33 @@ jobs:
       (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
       (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
       (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       pull-requests: write
       issues: write
       id-token: write
-      actions: read
+      actions: read # Required for Claude to read CI results on PRs
     steps:
       - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - name: Run Claude Code
         id: claude
-        uses: anthropics/claude-code-action@b76a0776ae74036e77cd11018083743453d7ad35 # v1
+        uses: anthropics/claude-code-action@v1
         with:
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+
+          # This is an optional setting that allows Claude to read CI results on PRs
           additional_permissions: |
             actions: read
+
+          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
+          # prompt: 'Update the pull request description to include a summary of changes.'
+
+          # Optional: Add claude_args to customize behavior and configuration
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
           claude_args: '--allowed-tools Bash(gh pr:*)'

.github/workflows/codeql.yml

@@ -1,21 +1,42 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
 name: "CodeQL"
 
 on:
-  workflow_call:
+  push:
+    branches: [ "dev" ]
+  pull_request:
+    branches: [ "dev" ]
   schedule:
     - cron: '33 17 * * 6'
 
-concurrency:
-  group: codeql-${{ github.repository }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
   analyze:
     name: Analyze (${{ matrix.language }})
-    runs-on: ubuntu-24.04
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
     permissions:
+      # required for all workflows
       security-events: write
+
+      # required to fetch internal or private CodeQL packs
       packages: read
+
+      # only required for workflows in private repositories
       actions: read
       contents: read
 
@@ -25,18 +46,47 @@ jobs:
         include:
         - language: python
           build-mode: none
-        - language: javascript-typescript
-          build-mode: none
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
     steps:
     - name: Checkout repository
-      uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      uses: actions/checkout@v4
 
+    # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
+      uses: github/codeql-action/analyze@v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/debian.yml

@@ -0,0 +1,210 @@
+name: Build Debian package
+
+on:
+  workflow_dispatch:
+  workflow_call:
+  push:
+    branches: ['**']
+    paths:
+      - 'pkg/debian/**'
+      - 'bin/build_deb.sh'
+      - 'bin/release_deb.sh'
+      - '.github/workflows/debian.yml'
+      - 'pyproject.toml'
+  # release trigger is handled by release.yml to avoid double-runs
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Get version
+        id: version
+        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
+
+      - name: Install nfpm
+        env:
+          NFPM_VERSION: "2.45.1"
+        run: |
+          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
+            | sudo tar -xz -C /usr/local/bin nfpm
+          nfpm --version
+
+      - name: Build .deb package
+        run: |
+          export VERSION="${{ steps.version.outputs.version }}"
+          export ARCH="${{ matrix.arch }}"
+          ./bin/build_deb.sh
+
+      - name: Verify .deb package contents
+        run: |
+          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
+          echo "=== Package info ==="
+          dpkg-deb --info "$DEB_FILE"
+          echo ""
+          echo "=== Package contents ==="
+          dpkg-deb --contents "$DEB_FILE"
+          echo ""
+          echo "=== Control fields ==="
+          dpkg-deb --field "$DEB_FILE"
+
+      - name: Upload .deb artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
+          path: dist/*.deb
+
+  test:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            runner: ubuntu-24.04
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: false
+
+      - name: Install build dependencies
+        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
+        with:
+          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
+          version: 1.0
+
+      - name: Build local wheel
+        run: |
+          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+          uv build --wheel --out-dir /tmp/wheels/
+
+      - name: Download .deb artifact
+        uses: actions/download-artifact@v4
+        with:
+          pattern: archivebox-*-${{ matrix.arch }}.deb
+          merge-multiple: true
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
+
+      - name: Pre-seed virtualenv with local wheel before dpkg install
+        run: |
+          # CI-only: pre-seed the venv with a local wheel so we test the
+          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
+          # here (matching the system dep installed above) to ensure the venv is
+          # created with the correct Python version. On real installs, install.sh
+          # handles this by preferring python3.13 and failing if < 3.13.
+          # When postinstall.sh runs during dpkg -i, it finds the venv already
+          # populated and just upgrades deps.
+          sudo mkdir -p /opt/archivebox
+          sudo python3.13 -m venv /opt/archivebox/venv
+          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
+          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
+          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
+
+      - name: Install .deb package
+        run: |
+          # dpkg install will run postinstall which creates the user,
+          # sets up systemd, and tries pip install (which finds it already installed)
+          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
+
+      - name: Verify archivebox is installed
+        run: |
+          which archivebox
+          archivebox version
+
+      - name: Verify wrapper script works
+        run: |
+          /usr/bin/archivebox version
+          /usr/bin/archivebox --help | head -5
+
+      - name: Test archivebox init as archivebox user
+        run: |
+          # The postinstall should have created the user
+          id archivebox
+          sudo mkdir -p /tmp/archivebox-test
+          sudo chown archivebox:archivebox /tmp/archivebox-test
+          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
+
+      - name: Test archivebox status
+        run: |
+          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
+
+      - name: Test archivebox add
+        run: |
+          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
+
+      - name: Verify systemd service file exists
+        run: |
+          test -f /usr/lib/systemd/system/archivebox.service
+          cat /usr/lib/systemd/system/archivebox.service
+
+  # Upload .deb to GitHub Release only after tests pass
+  release:
+    if: github.event_name == 'release' || github.event_name == 'workflow_call'
+    needs: [build, test]
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Determine release tag
+        id: tag
+        run: |
+          TAG="${{ github.event.release.tag_name }}"
+          if [ -z "$TAG" ]; then
+            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
+            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Download all .deb artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: archivebox-*.deb
+          merge-multiple: true
+
+      - name: Upload .deb to GitHub Release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          TAG="${{ steps.tag.outputs.tag }}"
+          # Verify the release exists before uploading
+          if ! gh release view "$TAG" >/dev/null 2>&1; then
+            echo "[!] No GitHub Release found for tag $TAG."
+            if [ -n "${{ github.event.release.tag_name }}" ]; then
+              echo "[X] This was triggered by a release event — the release should exist. Failing."
+              exit 1
+            fi
+            echo "[i] Skipping upload (workflow_dispatch without a release)."
+            echo "    Create a release first or trigger via the release event."
+            exit 0
+          fi
+          gh release upload "$TAG" *.deb --clobber

.github/workflows/deploy-publicsite.yml

@@ -1,96 +1,43 @@
-name: Publish website
+name: Deploy Publicsite to GitHub Pages
 
 on:
   push:
-    branches: [dev]
-  pull_request:
+    branches:
+      - dev
+    paths:
+      - publicsite/**
+      - .github/workflows/deploy-publicsite.yml
   workflow_dispatch:
-  workflow_run:
-    workflows: ['Capture publicsite screenshots']
-    types: [completed]
-    branches: [dev]
 
 permissions:
   contents: read
-  actions: read
+  pages: write
+  id-token: write
 
-env:
-  # Repository-specific output; all publication logic below is a local copy.
-  SITE_OUTPUT: _site
-  DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-
-# One production queue covers restore, build and deploy. PRs have separate queues.
 concurrency:
-  group: pages-${{ github.event_name == 'pull_request' && github.ref || 'production' }}
-  cancel-in-progress: false
+  group: github-pages-publicsite
+  cancel-in-progress: true
 
 jobs:
-  build:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    outputs:
-      revision: ${{ steps.source.outputs.revision }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event_name == 'pull_request' && github.ref || env.DEFAULT_BRANCH }}
-          persist-credentials: false
-      - name: Record website source revision
-        id: source
-        run: echo "revision=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
-
-      # BEGIN REPOSITORY-SPECIFIC BUILD
-      - uses: astral-sh/setup-uv@v7
-      - name: Restore latest complete screenshots
-        run: uv run --no-project python bin/restore_site_screenshots.py publicsite/screenshots
-        env:
-          GH_TOKEN: ${{ github.token }}
-      - name: Render current gallery layout without recapturing images
-        run: uv run --no-project python bin/generate_ui_screenshot_gallery.py refresh publicsite/screenshots/index.html
-      - name: Assemble website with the local base components
-        run: uv run --no-project python .github/pages/site.py render _site --source publicsite
-      # END REPOSITORY-SPECIFIC BUILD
-
-      - name: Verify website in Chromium
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/site-browser
-        run: |
-          # Resolve the same supported browser as captures; Playwright is the
-          # driver only and must not provision a separate browser/runtime.
-          ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
-          CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
-          uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
-          CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
-          export CHROME_BINARY
-          test -x "$CHROME_BINARY"
-          uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py "$SITE_OUTPUT" --evidence "$RUNNER_TEMP/site-evidence"
-      - name: Save website verification evidence
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: site-verification
-          path: ${{ runner.temp }}/site-evidence
-          if-no-files-found: warn
-          retention-days: 14
-
-      - name: Upload website
-        uses: actions/upload-pages-artifact@v3
-        with:
-          path: ${{ env.SITE_OUTPUT }}
-
   deploy:
-    if: github.event_name != 'pull_request'
-    needs: build
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      pages: write
-      id-token: write
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/configure-pages@v5
-      - uses: actions/deploy-pages@v4
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./publicsite
+
+      - name: Deploy to GitHub Pages
         id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/docker.yml

@@ -1,44 +1,27 @@
 name: Build Docker image
 
 on:
-  workflow_call:
-    inputs:
-      push_digests:
-        description: Push validated per-platform images by digest for the release workflow
-        required: false
-        default: false
-        type: boolean
-      full_tests:
-        description: Run storage, CLI, and size validation in addition to release smoke checks
-        required: false
-        default: true
-        type: boolean
   workflow_dispatch:
-    inputs:
-      source_ref:
-        description: Exact 40-character commit SHA to build
-        required: true
-        type: string
-      push_digests:
-        description: Push validated per-platform images by digest for the release workflow
-        required: false
-        default: true
-        type: boolean
-      full_tests:
-        description: Run storage, CLI, and size validation in addition to release smoke checks
-        required: false
-        default: true
-        type: boolean
+  workflow_call:
+  push:
+    branches:
+      - dev
+    tags:
+      - 'v*'
+  # pull_request:
 
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
   GHCR_IMAGE: ghcr.io/archivebox/archivebox
 
 permissions:
-  actions: read
   contents: read
   packages: write
 
+concurrency:
+  group: docker-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     name: build ${{ matrix.platform }}
@@ -49,94 +32,24 @@ jobs:
         include:
           - platform: linux/amd64
             runner: ubuntu-24.04
+            cache_scope: docker-amd64
             artifact_name: digest-linux-amd64
-            local_tag: archivebox-ci:amd64
           - platform: linux/arm64
             runner: ubuntu-24.04-arm
+            cache_scope: docker-arm64
             artifact_name: digest-linux-arm64
-            local_tag: archivebox-ci:arm64
     steps:
       - name: Checkout
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          ref: ${{ inputs.source_ref || github.sha }}
-
-      - name: Verify exact source commit
-        if: github.event_name == 'workflow_dispatch'
-        shell: bash
-        env:
-          SOURCE_REF: ${{ inputs.source_ref || github.sha }}
-        run: |
-          set -Eeuo pipefail
-          [[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
-          [[ "$(git rev-parse HEAD)" == "$SOURCE_REF" ]]
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Resolve Docker through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python - <<'PY'
-          import tomllib
-          from pathlib import Path
-
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          matches = [package for package in packages if package["name"] == "abxpkg"]
-          if len(matches) != 1:
-              raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}")
-          wheel = matches[0]["wheels"][0]
-          print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))
-          PY
-          )"
-          test -n "$ABXPKG_SPEC"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_SPEC=$ABXPKG_SPEC" >> "$GITHUB_ENV"
-          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-
-          docker_env="$(
-            uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries"
-          )"
-          JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$docker_env" >> "$GITHUB_ENV"
-
-          docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
-          UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv"
-          {
-            echo "DOCKER_BINARY=$docker_binary"
-            echo "JQ_BINARY=$JQ_BINARY"
-            echo "UV_BINARY=$UV_BINARY"
-          } >> "$GITHUB_ENV"
-          for binary in "$docker_binary" "$JQ_BINARY" "$UV_BINARY"; do
-            test -L "$binary"
-            test -x "$binary"
-          done
-          "$docker_binary" --version
+        uses: actions/checkout@v4
+        # with:
+        #   submodules: true
+        #   fetch-depth: 1
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+        uses: docker/setup-buildx-action@v3
         with:
+          version: latest
           install: true
 
       - name: Builder instance name
@@ -145,50 +58,15 @@ jobs:
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
-      - name: Verify published abx-dl image
-        id: abx_dl_image
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABX_DL_VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import tomllib
-          from pathlib import Path
-
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          matches = [package["version"] for package in packages if package["name"] == "abx-dl"]
-          if len(matches) != 1:
-              raise SystemExit(f"Expected one locked abx-dl package, found {len(matches)}")
-          print(matches[0])
-          PY
-          )"
-          test -n "$ABX_DL_VERSION"
-          ABX_DL_REFERENCE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
-          INSPECT_OUTPUT="$("$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_REFERENCE")"
-          printf '%s\n' "$INSPECT_OUTPUT"
-          ABX_DL_DIGEST="$(INSPECT_OUTPUT="$INSPECT_OUTPUT" "$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import os
-
-          digest = next(
-              (fields[1] for line in os.environ["INSPECT_OUTPUT"].splitlines() if (fields := line.split()) and fields[0] == "Digest:"),
-              "",
-          )
-          print(digest)
-          PY
-          )"
-          [[ "$ABX_DL_DIGEST" == sha256:* ]]
-          ABX_DL_IMAGE="${ABX_DL_REFERENCE%%@*}@${ABX_DL_DIGEST}"
-          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
-
       - name: Login to Docker Hub
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        if: inputs.push_digests
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        if: inputs.push_digests
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -199,416 +77,185 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
-          import tomllib
+          VERSION="$(python3 - <<'PY'
           from pathlib import Path
+          import re
 
-          print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])
+          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
+          if not match:
+              raise SystemExit("Failed to read version from pyproject.toml")
+          print(match.group(1))
           PY
           )"
-          test -n "$VERSION"
 
           {
             echo 'labels<<EOF'
             echo "org.opencontainers.image.version=${VERSION}"
-            echo "org.opencontainers.image.revision=${{ inputs.source_ref || github.sha }}"
+            echo "org.opencontainers.image.revision=${GITHUB_SHA}"
             echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
-            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
             echo 'EOF'
             echo "version=${VERSION}"
           } >> "$GITHUB_OUTPUT"
 
-          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
+          echo "[+] Building ${{ matrix.platform }} for ${VERSION}"
 
       - name: Build and push digest
         id: docker_build
-        if: inputs.push_digests
-        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
+        uses: docker/build-push-action@v6
         with:
           context: ./
           file: ./Dockerfile
           builder: ${{ steps.buildx.outputs.name }}
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: |
+            ${{ env.DOCKERHUB_IMAGE }}
+            ${{ env.GHCR_IMAGE }}
           labels: ${{ steps.docker_meta.outputs.labels }}
-          build-args: |
-            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
-          pull: true
+          cache-from: type=gha,scope=${{ matrix.cache_scope }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
           platforms: ${{ matrix.platform }}
-          # Keep the released image as the one cache authority. A second GHA
-          # cache can hold gzip variants of identical layers, and BuildKit may
-          # prefer those over :dev's zstd blobs. That counter-intuitively makes
-          # a cache hit larger than the released image. :dev advances only
-          # after both architectures pass the release checks.
-          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
-          # Inline metadata reuses the released compressed blobs directly;
-          # force-compression would recompress the whole ~740 MiB image. The
-          # compressed-size gate below remains the regression guard.
-          cache-to: |
-            ${{ inputs.push_digests && 'type=inline' || '' }}
-          outputs: |
-            ${{ inputs.full_tests && format('type=oci,dest={0}/archivebox-image.tar,compression=zstd,compression-level=3', runner.temp) || '' }}
-            ${{ inputs.push_digests && format('type=image,"name={0},{1}",push-by-digest=true,name-canonical=true,push=true,compression=zstd,compression-level=3', env.DOCKERHUB_IMAGE, env.GHCR_IMAGE) || '' }}
-
-      - name: Build pull request image
-        id: docker_build_pr
-        if: ${{ !inputs.push_digests }}
-        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
-        with:
-          context: ./
-          file: ./Dockerfile
-          builder: ${{ steps.buildx.outputs.name }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-          build-args: |
-            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-            ARCHIVEBOX_COMMIT_HASH=${{ inputs.source_ref || github.sha }}
-          pull: true
-          platforms: ${{ matrix.platform }}
-          # Test the same layer lineage users receive. Reusing :dev avoids a
-          # second cache whose compression can drift from the released image.
-          cache-from: type=registry,ref=${{ env.DOCKERHUB_IMAGE }}:dev
-          outputs: |
-            type=docker,name=${{ matrix.local_tag }}
-            type=oci,dest=${{ runner.temp }}/archivebox-image.tar
+          outputs: type=image,push-by-digest=true,name-canonical=true,push=true
 
       - name: Image digest
-        if: inputs.push_digests
         run: echo ${{ steps.docker_build.outputs.digest }}
 
-      - name: Verify optional agent install with a read-only browser profile
-        shell: bash
-        env:
-          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: |
-          set -Eeuo pipefail
-          PROFILE_DIR="$(mktemp -d)"
-          # shellcheck disable=SC2016
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' \
-            --volume "$PROFILE_DIR:/home/archivebox/.config/chromium:ro" \
-            "$IMAGE" bash -c '
-              set -Eeuo pipefail
-              test -w "$XDG_CONFIG_HOME"
-              test ! -w "$XDG_CONFIG_HOME/chromium"
-              uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_archivebox_install; verify_archivebox_install()"
-            '
-
-      - name: Validate exact built image version and commit
-        if: inputs.full_tests
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}"
-          SHORT_SHA="${{ (inputs.source_ref || github.sha) }}"
-          SHORT_SHA="${SHORT_SHA::7}"
-
-          DATA_DIR="$(mktemp -d)"
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
-          ARCHIVEBOX_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version)"
-          printf '%s\n' "$ARCHIVEBOX_VERSION_OUTPUT"
-          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
-          IMAGE_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt)"
-          ARCHIVEBOX_VERSION_OUTPUT="$ARCHIVEBOX_VERSION_OUTPUT" IMAGE_VERSION_OUTPUT="$IMAGE_VERSION_OUTPUT" \
-            "$UV_BINARY" run --no-cache --no-project python - \
-            "ArchiveBox v${{ steps.docker_meta.outputs.version }}" \
-            "COMMIT_HASH=${SHORT_SHA}" \
-            "COMMIT_HASH=${GITHUB_SHA}" <<'PY'
-          import os
-          import sys
-
-          version_output = os.environ["ARCHIVEBOX_VERSION_OUTPUT"]
-          image_version_lines = os.environ["IMAGE_VERSION_OUTPUT"].splitlines()
-          if sys.argv[1] not in version_output:
-              raise SystemExit(f"Missing version marker: {sys.argv[1]}")
-          if sys.argv[2] not in version_output:
-              raise SystemExit(f"Missing short commit marker: {sys.argv[2]}")
-          if sys.argv[3] not in image_version_lines:
-              raise SystemExit(f"Missing exact commit marker: {sys.argv[3]}")
-          PY
-
-          "$DOCKER_BINARY" run --rm --user 911:911 --platform '${{ matrix.platform }}' --entrypoint /venv/bin/python3 "$IMAGE" -c \
-            'import json, pathlib, stat; cache_file = "/opt/archivebox/lib/env/derived.env"; raw = open(cache_file).read().strip(); records = json.loads(raw[len("ABXPKG_DERIVED_CACHE=\x27"):-1]); projections = [projection for record in records.values() for projection in record.get("request_exec_projections", {}).values()]; plans = [projection["validation"] for projection in projections]; fingerprints = [fingerprint for plan in plans for fingerprint in plan["fingerprint"]]; actual = lambda fingerprint: (lambda path, value: {"path": str(path.resolve(strict=False)), "size": value.st_size, "mtime_ns": value.st_mtime_ns, "mode": stat.S_IMODE(value.st_mode), "euid": value.st_uid})(pathlib.Path(fingerprint["path"]), pathlib.Path(fingerprint["path"]).stat()); mismatches = [(fingerprint, actual(fingerprint)) for fingerprint in fingerprints if fingerprint != actual(fingerprint)]; assert projections and {plan.get("euid") for plan in plans} == {911}; assert all(not record.get("script_exec_plans") for record in records.values()); assert "ABXPKG_TMP_CACHE_DIR" not in json.dumps(projections); assert not mismatches, mismatches[:3]'
-
-      - name: Validate compressed candidate size
-        if: inputs.push_digests
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          DIGEST='${{ steps.docker_build.outputs.digest }}'
-          [[ "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]
-          MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${DIGEST}")"
-          # Provenance wraps the real platform image and its attestation in an
-          # OCI index. Measure the image manifest, not the index/attestation.
-          IMAGE_DIGEST="$(MANIFEST="$MANIFEST" PLATFORM='${{ matrix.platform }}' "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); platform_os, platform_arch = os.environ["PLATFORM"].split("/")[:2]; print(next((item["digest"] for item in manifest.get("manifests", []) if item.get("platform", {}).get("os") == platform_os and item.get("platform", {}).get("architecture") == platform_arch), ""))')"
-          if [[ -n "$IMAGE_DIGEST" ]]; then
-            MANIFEST="$("$DOCKER_BINARY" buildx imagetools inspect --raw "${DOCKERHUB_IMAGE}@${IMAGE_DIGEST}")"
-          fi
-          COMPRESSED_SIZE="$(MANIFEST="$MANIFEST" "$UV_BINARY" run --no-cache --no-project python -c 'import json, os; manifest = json.loads(os.environ["MANIFEST"]); print(manifest["config"]["size"] + sum(layer["size"] for layer in manifest["layers"]))')"
-          LIMIT=$((1025 * 1024 * 1024))
-          echo "${DIGEST} compressed_size=$((COMPRESSED_SIZE / 1024 / 1024)) MiB limit=1025 MiB"
-          (( COMPRESSED_SIZE < LIMIT ))
-
-      - name: Resolve real Docker CLI test dependencies through abxpkg
-        id: mount_dependencies
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mount_env="$(
-            uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-              --install \
-              --json \
-              --lib="$ABXPKG_LIB_DIR" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries" \
-              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:mount_validation_binaries"
-          )"
-          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
-            <<< "$mount_env" >> "$GITHUB_ENV"
-
-          for binary_name in bash chmod curl exportfs mkdir mount.cifs mount.nfs smbd sudo systemctl umount; do
-            binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
-            test -L "$binary"
-            test -x "$binary"
-          done
-
-      - name: Diagnose failed mount dependency installation
-        if: failure() && steps.mount_dependencies.conclusion == 'failure'
-        shell: bash
-        run: |
-          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
-          sudo journalctl --since '-10 minutes' --no-pager \
-            -u smbd -u nmbd -u samba-ad-dc -u nfs-server
-          ps -eo pid,ppid,stat,etimes,wchan:32,comm
-
-      - name: Validate Docker mounts, ports, CLI and compose concurrently
-        if: inputs.full_tests && matrix.platform == 'linux/amd64'
-        shell: bash
-        env:
-          ARCHIVEBOX_IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: |
-          set -Eeuo pipefail
-          # These checks own separate collections, Compose projects and ports.
-          # Subshells keep mount cleanup traps and environment changes isolated.
-          validate_mounts() (
-            set -Eeuo pipefail
-            NFS_SOURCE="/srv/archivebox-ci-nfs"
-            SMB_SOURCE="/srv/archivebox-ci-smb"
-            NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
-            SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
-            mounted_nfs=0
-            mounted_smb=0
-
-            # Invoked by the subshell EXIT trap.
-            # shellcheck disable=SC2329
-            cleanup() {
-              local original_status=$? cleanup_status=0
-              trap - EXIT
-              if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
-                cleanup_status=1
-              fi
-              if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
-                cleanup_status=1
-              fi
-              if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
-                cleanup_status=1
-              fi
-              if [[ "$original_status" != "0" ]]; then
-                exit "$original_status"
-              fi
-              exit "$cleanup_status"
-            }
-            trap cleanup EXIT
-
-            "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-            "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
-            "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
-            "$SUDO_BINARY" "$BASH_BINARY" -c \
-              "printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
-            "$SUDO_BINARY" "$BASH_BINARY" -c \
-              "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
-
-            "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
-            "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
-            "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
-            "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
-            "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
-
-            "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
-            mounted_nfs=1
-            "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
-              -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
-            mounted_smb=1
-
-            NFS_TEST_DIR="$NFS_MOUNT" \
-            SMB_TEST_DIR="$SMB_MOUNT" \
-            IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
-              "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
-          )
-          validate_cli() (
-            set -Eeuo pipefail
-            "$MKDIR_BINARY" "${{ github.workspace }}/data"
-            "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
-
-            "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
-            "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
-
-            "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
-            # The argument form above exercises the complete default pipeline.
-            # Keep a real capture for stdin parsing without repeating every plugin.
-            echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add --plugins=wget,hashes
-
-            archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
-            [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
-            [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
-
-            "$DOCKER_BINARY" compose run --rm archivebox init
-            "$DOCKER_BINARY" compose up -d --wait
-            "$DOCKER_BINARY" network connect archivebox_default www-nginx
-            index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-            admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
-            [[ "$index_html" == *ArchiveBox* ]]
-            [[ "$admin_js" == *window.django* ]]
-
-            "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
-            index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
-            [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
-            "$DOCKER_BINARY" compose down
-          )
-          validate_ports() (
-            IMAGE="$ARCHIVEBOX_IMAGE" "$BASH_BINARY" bin/validate_docker_ports.sh
-          )
-
-          checks=(mounts ports cli)
-          pids=()
-          validate_mounts > "$RUNNER_TEMP/docker-mounts.log" 2>&1 &
-          pids+=("$!")
-          validate_ports > "$RUNNER_TEMP/docker-ports.log" 2>&1 &
-          pids+=("$!")
-          validate_cli > "$RUNNER_TEMP/docker-cli.log" 2>&1 &
-          pids+=("$!")
-          status=0
-          for index in "${!checks[@]}"; do
-            check="${checks[$index]}"
-            echo "::group::Docker $check validation"
-            if wait "${pids[$index]}"; then
-              echo "PASS: Docker $check validation"
-            else
-              echo "::error::Docker $check validation failed"
-              status=1
-            fi
-            cat "$RUNNER_TEMP/docker-$check.log"
-            echo "::endgroup::"
-          done
-          exit "$status"
-
-      - name: Validate Docker port compatibility
-        if: ${{ !(inputs.full_tests && matrix.platform == 'linux/amd64') }}
-        shell: bash
-        env:
-          IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
-        run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
-
-      - name: Validate compressed image size
-        if: inputs.full_tests && !inputs.push_digests
-        env:
-          BUILD_METADATA: ${{ steps.docker_build_pr.outputs.metadata }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          LOCAL_IMAGE_ID="$("$DOCKER_BINARY" image inspect '${{ matrix.local_tag }}' --format '{{.Id}}')"
-          "$UV_BINARY" run --no-cache --no-project python - \
-            '${{ runner.temp }}/archivebox-image.tar' \
-            '${{ steps.docker_build_pr.outputs.digest }}' \
-            "$LOCAL_IMAGE_ID" \
-            "$((1025 * 1024 * 1024))" <<'PY'
-          from hashlib import sha256
-          import json
-          import os
-          import sys
-          import tarfile
-
-          archive_path, pushed_digest, local_image_id, limit_text = sys.argv[1:]
-          limit = int(limit_text)
-          metadata = json.loads(os.environ["BUILD_METADATA"])
-          if metadata["containerimage.digest"] != pushed_digest:
-              raise SystemExit("Build push transaction digest does not match the action output")
-          if not pushed_digest.startswith("sha256:") or len(pushed_digest) != 71:
-              raise SystemExit(f"Invalid pushed digest: {pushed_digest}")
-
-          with tarfile.open(archive_path) as archive:
-              def read_blob(descriptor):
-                  algorithm, digest = descriptor["digest"].split(":", 1)
-                  if algorithm != "sha256":
-                      raise SystemExit(f"Unsupported OCI digest: {descriptor['digest']}")
-                  member = archive.extractfile(f"blobs/sha256/{digest}")
-                  if member is None:
-                      raise SystemExit(f"Missing OCI blob: {descriptor['digest']}")
-                  blob = member.read()
-                  if len(blob) != descriptor["size"]:
-                      raise SystemExit(f"OCI size mismatch: {descriptor['digest']}")
-                  if sha256(blob).hexdigest() != digest:
-                      raise SystemExit(f"OCI digest mismatch: {descriptor['digest']}")
-                  return blob
-
-              index_member = archive.extractfile("index.json")
-              if index_member is None:
-                  raise SystemExit("OCI export is missing index.json")
-              index = json.load(index_member)
-
-              manifests = []
-
-              def collect(descriptor):
-                  document = json.loads(read_blob(descriptor))
-                  media_type = descriptor["mediaType"]
-                  if "image.index" in media_type or "manifest.list" in media_type:
-                      for child in document["manifests"]:
-                          collect(child)
-                  elif "image.manifest" in media_type:
-                      config_type = document["config"]["mediaType"]
-                      if "image.config" in config_type or "container.image" in config_type:
-                          manifests.append(document)
-
-              for root in index["manifests"]:
-                  collect(root)
-              if len(manifests) != 1:
-                  raise SystemExit(f"Expected one runnable image manifest, found {len(manifests)}")
-
-              manifest = manifests[0]
-              config_digest = manifest["config"]["digest"]
-              if metadata["containerimage.config.digest"] != config_digest:
-                  raise SystemExit("OCI config digest does not match the pushed build result")
-              if local_image_id != config_digest:
-                  raise SystemExit("Locally tested image does not match the OCI build result")
-              descriptors = [manifest["config"], *manifest["layers"]]
-              for descriptor in descriptors:
-                  read_blob(descriptor)
-              total = sum(descriptor["size"] for descriptor in descriptors)
-
-          print(
-              f"{pushed_digest} compressed_size={total // 1024 // 1024} MiB "
-              f"limit={limit // 1024 // 1024} MiB",
-          )
-          if total > limit:
-              raise SystemExit(f"{pushed_digest} is over the compressed size limit")
-          PY
-
       - name: Export digest
-        if: inputs.push_digests
-        env:
-          ABX_DL_IMAGE_REF: ${{ steps.abx_dl_image.outputs.image }}
-          ARTIFACT_NAME: ${{ matrix.artifact_name }}
         shell: bash
         run: |
           set -Eeuo pipefail
           mkdir -p /tmp/digests
           digest="${{ steps.docker_build.outputs.digest }}"
           touch "/tmp/digests/${digest#sha256:}"
-          printf '%s\n' '${{ inputs.source_ref || github.sha }}' > /tmp/digests/source-ref
-          printf '%s\n' "$ABX_DL_IMAGE_REF" > "/tmp/digests/abx-dl-image-$ARTIFACT_NAME"
 
       - name: Upload digest
-        if: inputs.push_digests
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.artifact_name }}
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1
+
+  publish:
+    name: publish multiarch tags
+    runs-on: ubuntu-24.04
+    needs:
+      - build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: latest
+          install: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digest-*
+          merge-multiple: true
+
+      - name: Collect Docker tags
+        id: docker_meta
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$(python3 - <<'PY'
+          from pathlib import Path
+          import re
+
+          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
+          if not match:
+              raise SystemExit("Failed to read version from pyproject.toml")
+          print(match.group(1))
+          PY
+          )"
+          SHORT_SHA="${GITHUB_SHA::8}"
+
+          {
+            echo 'dockerhub_tags<<EOF'
+            echo "${DOCKERHUB_IMAGE}:dev"
+            echo "${DOCKERHUB_IMAGE}:${VERSION}"
+            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+            echo 'EOF'
+            echo 'ghcr_tags<<EOF'
+            echo "${GHCR_IMAGE}:dev"
+            echo "${GHCR_IMAGE}:${VERSION}"
+            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+            echo 'EOF'
+            echo "version=${VERSION}"
+            echo "short_sha=${SHORT_SHA}"
+          } >> "$GITHUB_OUTPUT"
+
+          echo "[+] Publishing Docker Hub tags:"
+          printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+          echo "[+] Publishing GHCR tags:"
+          printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+
+      - name: Create Docker Hub manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
+          [[ "${#DIGESTS[@]}" -gt 0 ]]
+
+          TAG_ARGS=()
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
+          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
+
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do
+            REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}")
+          done
+
+          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Create GHCR manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
+          [[ "${#DIGESTS[@]}" -gt 0 ]]
+
+          TAG_ARGS=()
+          while IFS= read -r tag; do
+            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
+          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
+
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do
+            REFS+=("${GHCR_IMAGE}@sha256:${digest}")
+          done
+
+          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Inspect published images
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+          docker buildx imagetools inspect "${GHCR_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
+
+      - name: Update README
+        uses: peter-evans/dockerhub-description@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: archivebox/archivebox

.github/workflows/docs-links.yml

@@ -1,38 +0,0 @@
-name: Documentation link checks
-
-on:
-  schedule:
-    - cron: '17 10 * * *' # Daily at 10:17 UTC, away from the busy top of the hour.
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-env:
-  PYTHONIOENCODING: utf-8
-
-jobs:
-  links:
-    name: Documentation links and anchors
-    runs-on: ubuntu-24.04
-    timeout-minutes: 5
-    permissions:
-      contents: read
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-      - name: List tracked documentation and source inputs
-        run: |
-          git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > "$RUNNER_TEMP/link-inputs.txt"
-      - uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
-        with:
-          lycheeVersion: v0.24.2
-          args: >-
-            --config .github/lychee.toml
-            --root-dir "${{ github.workspace }}"
-            --files-from "${{ runner.temp }}/link-inputs.txt"
-          fail: true
-          failIfEmpty: true

.github/workflows/docs.yml

@@ -1,195 +0,0 @@
-name: Documentation tests
-
-on:
-  schedule:
-    - cron: '7 5,17 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-env:
-  PYTHONIOENCODING: utf-8
-  USE_COLOR: "False"
-  SHOW_PROGRESS: "False"
-
-jobs:
-  changes:
-    runs-on: ubuntu-24.04
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      key: ${{ steps.check.outputs.key }}
-    steps:
-      - id: check
-        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
-
-  discover:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    name: Discover documentation examples
-    runs-on: ubuntu-24.04
-    timeout-minutes: 5
-    outputs:
-      matrix: ${{ steps.inventory.outputs.matrix }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-      - name: Install ArchiveBox development dependencies
-        run: uv sync --no-cache --locked --dev
-      - name: Resolve documentation tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
-        run: |
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          uv run --no-cache --no-sync abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries"
-          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
-          mmdc_binary="$ABXPKG_LIB_DIR/env/bin/mmdc"
-          nginx_binary="$ABXPKG_LIB_DIR/env/bin/nginx"
-          test -L "$bash_binary"
-          test -L "$mmdc_binary"
-          test -L "$nginx_binary"
-          test -x "$bash_binary"
-          test -x "$mmdc_binary"
-          test -x "$nginx_binary"
-          {
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-            echo "BASH_BINARY=$bash_binary"
-          } >> "$GITHUB_ENV"
-      - id: inventory
-        name: Validate the complete authored documentation inventory
-        run: |
-          matrix="$(uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py matrix)"
-          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
-
-  examples:
-    name: docs/${{ matrix.name }}
-    needs: discover
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 15
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          submodules: true
-          fetch-depth: 1
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        with:
-          python-version: "3.13"
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-      - name: Install ArchiveBox
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        run: uv sync --no-cache --locked --dev
-      - name: Resolve documentation runtime through abxpkg
-        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
-        run: |
-          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
-          dependency_sources=(
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
-          )
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
-          fi
-          if [[ '${{ matrix.name }}' == ubuntu-AGENTS || '${{ matrix.name }}' == ubuntu-skills-archivebox-SKILL || '${{ matrix.name }}' == ubuntu-docs-Configuration || '${{ matrix.name }}' == ubuntu-docs-Setting-up-Authentication ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          fi
-          if [[ '${{ matrix.environment }}' == docker ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
-          elif [[ '${{ matrix.environment }}' == macos ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")
-          fi
-          uv run --no-cache --no-sync abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            "${dependency_sources[@]}"
-          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
-          python_binary="$ABXPKG_LIB_DIR/env/bin/python"
-          uv_binary="$ABXPKG_LIB_DIR/env/bin/uv"
-          test -L "$bash_binary"
-          test -x "$bash_binary"
-          test -L "$python_binary"
-          test -x "$python_binary"
-          test -L "$uv_binary"
-          test -x "$uv_binary"
-          {
-            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
-            echo "BASH_BINARY=$bash_binary"
-            echo "PYTHON_BINARY=$python_binary"
-            echo "UV_BINARY=$uv_binary"
-          } >> "$GITHUB_ENV"
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            sudo_binary="$ABXPKG_LIB_DIR/env/bin/sudo"
-            test -L "$sudo_binary"
-            test -x "$sudo_binary"
-            echo "SUDO_BINARY=$sudo_binary" >> "$GITHUB_ENV"
-          fi
-          if [[ '${{ matrix.environment }}' != macos ]]; then
-            echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
-          fi
-          if [[ '${{ matrix.environment }}' == docker ]]; then
-            docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
-            test -L "$docker_binary"
-            test -x "$docker_binary"
-            echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV"
-          elif [[ '${{ matrix.environment }}' == macos ]]; then
-            test -L "$ABXPKG_LIB_DIR/env/bin/brew"
-            test -x "$ABXPKG_LIB_DIR/env/bin/brew"
-          fi
-      - name: Build the exact checkout's documentation image
-        if: ${{ matrix.environment == 'docker' }}
-        run: |
-          ABX_DL_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(dep.removeprefix("abx-dl==") for dep in tomllib.load(open("pyproject.toml", "rb"))["project"]["dependencies"] if dep.startswith("abx-dl==")))')"
-          "$DOCKER_BINARY" build --build-arg "ABX_DL_IMAGE=archivebox/abx-dl:${ABX_DL_VERSION}" --tag archivebox/archivebox:dev .
-      - name: Run the Ubuntu, Docker, or macOS documentation lane
-        if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
-        run: uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
-      - name: Run the root documentation lane
-        if: ${{ matrix.environment == 'root' }}
-        run: |
-          "$SUDO_BINARY" \
-            --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
-            "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
-      - name: Diagnose failed Linux package installation
-        if: failure() && runner.os == 'Linux'
-        run: |
-          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
-          apt-config dump Acquire::Retries Acquire::http::Timeout Acquire::https::Timeout
-          ps -eo pid,ppid,stat,etimes,wchan:32,comm
-
-  complete:
-    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
-    needs: [changes, discover, examples]
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - run: echo 'Scheduled work completed successfully.'

.github/workflows/duplicate-issue-detection.yml

@@ -0,0 +1,59 @@
+name: Duplicate Issue Detection
+
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  check-duplicates:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Install opencode
+        run: curl -fsSL https://opencode.ai/install | bash
+
+      - name: Check for duplicate issues
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OPENCODE_PERMISSION: |
+            {
+              "bash": {
+                "gh issue*": "allow",
+                "gh pr*": "allow",
+                "*": "deny"
+              },
+              "webfetch": "allow"
+            }
+        run: |
+          opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created:
+
+          Issue number:
+          ${{ github.event.issue.number }}
+
+          Lookup this issue and search through existing issues and PRs (excluding #${{ github.event.issue.number }}) in this repository to find any potential duplicates of this new issue.
+          Consider:
+          1. Similar titles or descriptions
+          2. Same error messages or symptoms
+          3. Related functionality or components
+          4. Similar feature requests
+
+          If you find any potential duplicates, please comment on the new issue with:
+          - A brief explanation of why it might be a duplicate
+          - Links to the potentially duplicate issues or PRs
+          - A suggestion to check those issues first
+
+          Use this format for the comment:
+          'This issue might be a duplicate of existing issues. Please check:
+          - #[issue_number]: [brief description of similarity]
+
+          Feel free to ignore if none of these address your specific case.'
+
+          If no clear duplicates are found, do not comment."

.github/workflows/homebrew.yml

@@ -0,0 +1,229 @@
+name: Build Homebrew formula
+
+on:
+  workflow_dispatch:
+  workflow_call:
+  push:
+    branches: ['**']
+    paths:
+      - 'brew_dist/**'
+      - 'bin/build_brew.sh'
+      - 'bin/release_brew.sh'
+      - '.github/workflows/homebrew.yml'
+      - 'pyproject.toml'
+  # release trigger is handled by release.yml to avoid double-runs
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-24.04]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Get version
+        id: version
+        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
+
+      - name: Validate formula template syntax
+        run: ruby -c brew_dist/archivebox.rb
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: false
+
+      - name: Install build dependencies (Linux)
+        if: runner.os == 'Linux'
+        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
+        with:
+          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
+          version: 1.0
+
+      - name: Build local sdist
+        run: |
+          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+          uv build --sdist --out-dir /tmp/sdist/
+
+      - name: Generate formula from local sdist
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
+          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
+
+          # Install archivebox + poet into a temp venv to generate resource stanzas
+          python3 -m venv /tmp/poet-venv
+          source /tmp/poet-venv/bin/activate
+          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
+          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
+          RESOURCES="$(poet archivebox)"
+          deactivate
+
+          # For CI: use file:// URL pointing to local sdist
+          # For release: this gets overridden with the PyPI URL
+          SDIST_URL="file://${SDIST_PATH}"
+
+          cat > /tmp/archivebox.rb << RUBY
+class Archivebox < Formula
+  include Language::Python::Virtualenv
+
+  desc "Self-hosted internet archiving solution"
+  homepage "https://github.com/ArchiveBox/ArchiveBox"
+  url "${SDIST_URL}"
+  sha256 "${SDIST_SHA256}"
+  license "MIT"
+
+  depends_on "python@3.13"
+
+${RESOURCES}
+
+  def install
+    virtualenv_install_with_resources
+  end
+
+  def post_install
+    data_dir = var/"archivebox"
+    data_dir.mkpath
+    ENV["DATA_DIR"] = data_dir.to_s
+    system bin/"archivebox", "init"
+  end
+
+  def caveats
+    <<~EOS
+      ArchiveBox data is stored in:
+        #{var}/archivebox
+
+      To start archiving, run:
+        cd #{var}/archivebox && archivebox add 'https://example.com'
+
+      To start the web UI:
+        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
+    EOS
+  end
+
+  test do
+    assert_match version.to_s, shell_output("#{bin}/archivebox version")
+  end
+end
+RUBY
+
+          echo "[√] Generated formula:"
+          ruby -c /tmp/archivebox.rb
+          cat /tmp/archivebox.rb
+
+      - name: Install Homebrew (Linux only)
+        if: runner.os == 'Linux'
+        run: |
+          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
+          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
+          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
+          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
+          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
+          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
+
+      - name: Install brew dependencies
+        run: brew install python@3.13
+
+      - name: Install archivebox via brew from local formula
+        run: |
+          brew install --build-from-source --verbose /tmp/archivebox.rb
+
+      - name: Verify archivebox version
+        run: archivebox version
+
+      - name: Test archivebox init
+        run: |
+          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
+          archivebox init
+
+      - name: Test archivebox status
+        run: |
+          cd /tmp/archivebox-test
+          archivebox status
+
+  # On release only: generate the real formula with PyPI URL and push to tap
+  release:
+    if: github.event_name == 'release' || github.event_name == 'workflow_call'
+    needs: build-and-test
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Get version
+        id: version
+        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Wait for PyPI package availability
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
+          for i in $(seq 1 30); do
+            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
+              echo "[√] archivebox==${VERSION} is available on PyPI"
+              break
+            fi
+            if [ "$i" -eq 30 ]; then
+              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
+              break
+            fi
+            echo "    Attempt $i/30 - not yet available, waiting 30s..."
+            sleep 30
+          done
+
+      - name: Generate release formula via build_brew.sh
+        run: ./bin/build_brew.sh
+
+      - name: Test formula install
+        run: |
+          brew install --build-from-source brew_dist/archivebox.rb
+          archivebox version
+
+      - name: Upload formula artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: archivebox.rb
+          path: brew_dist/archivebox.rb
+
+      - name: Push to homebrew-archivebox tap
+        env:
+          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap
+
+          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
+            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb
+
+          cd /tmp/tap
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add -A
+          if git diff --cached --quiet; then
+            echo "[i] No changes to formula, skipping push."
+          else
+            git commit -m "Update archivebox to v${VERSION}"
+            git push origin HEAD
+            echo "[√] Formula pushed to homebrew-archivebox tap"
+          fi

.github/workflows/lint.yml

@@ -1,38 +1,46 @@
 name: Run linters
 
 on:
-  workflow_call:
+  workflow_dispatch:
+  push:
 
 env:
   UV_NO_SOURCES: "1"
+  PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
 
 jobs:
   lint:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
       - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/setup-python@v5
         with:
           python-version: "3.13"
           architecture: x64
 
       - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        uses: astral-sh/setup-uv@v6
         with:
-          version: "0.11.3"
           enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Verify checked dependency lock
-        run: uv lock --no-cache --check
 
       - name: Install dependencies with uv
-        run: uv sync --no-cache --locked --dev --extra sonic --extra debug
+        run: |
+          uv sync --all-extras --all-groups --no-sources --no-cache
+          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
 
       - name: Run prek
-        run: uv run --no-cache --no-sync --no-sources prek run --all-files
+        run: uv run --no-sync --no-sources prek run --all-files

.github/workflows/macos.yml

@@ -1,48 +0,0 @@
-name: macOS compatibility
-
-on:
-  schedule:
-    - cron: '51 2,10,18 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-jobs:
-  changes:
-    runs-on: ubuntu-24.04
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      key: ${{ steps.check.outputs.key }}
-    steps:
-      - id: check
-        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
-
-  install-cli:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    uses: ./.github/workflows/test.yml
-    with:
-      platform: macos
-    secrets: inherit
-
-  python-artifacts:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    uses: ./.github/workflows/pip.yml
-    with:
-      platforms: '["macos-15"]'
-    secrets: inherit
-
-  complete:
-    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
-    needs: [changes, install-cli, python-artifacts]
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - run: echo 'Scheduled macOS coverage completed successfully.'

.github/workflows/pip.yml

@@ -1,248 +1,69 @@
 name: Build Pip package
 
 on:
+  workflow_dispatch:
   workflow_call:
-    inputs:
-      platforms:
-        description: Runner matrix for package compatibility
-        required: false
-        default: '["ubuntu-24.04"]'
-        type: string
-      full_tests:
-        description: Run the full package compatibility matrix instead of one release smoke check
-        required: false
-        default: true
-        type: boolean
+  push:
+    branches:
+      - '**'
+    tags:
+      - 'v*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
-  UV_VERSION: "0.11.3"
+  PYTHON_VERSION: "3.13"
 
 jobs:
   build:
-    name: build distributions
+    permissions:
+      id-token: write
+
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
 
-      - name: Set up Python 3.13
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
         with:
-          python-version: "3.13"
+          enable-cache: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
           architecture: x64
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+      - name: APT install archivebox dev + run dependencies
+        uses: awalsh128/cache-apt-pkgs-action@latest
         with:
-          version: ${{ env.UV_VERSION }}
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
 
-      - name: Build every distribution
+      - name: UV install archivebox dev + run sub-dependencies
+        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
+
+      - name: UV build archivebox and archivebox/pkgs/* packages
         run: |
-          set -Eeuo pipefail
-          uv build --no-cache --all-packages --no-sources --out-dir dist --clear
+          uv build --all
 
-          shopt -s nullglob
-          artifacts=(dist/*)
-          wheels=(dist/archivebox-*.whl)
-          sdists=(dist/archivebox-*.tar.gz)
-          [[ "${#artifacts[@]}" -gt 0 ]]
-          [[ "${#wheels[@]}" -eq 1 ]]
-          [[ "${#sdists[@]}" -eq 1 ]]
-          printf '%s\n' "$GITHUB_SHA" > dist/COMMIT_SHA
-          uv run --no-cache --no-project python - <<'PY'
-          import hashlib
-          from pathlib import Path
+      - name: Publish new package wheels and sdists to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # ignore when publish to PyPI fails due to duplicate tag
+        continue-on-error: true
+        with:
+          password: ${{ secrets.PYPI_PAT_SECRET }}
 
-          root = Path("dist")
-          wheels = list(root.glob("archivebox-*.whl"))
-          sdists = list(root.glob("archivebox-*.tar.gz"))
-          if len(wheels) != 1 or len(sdists) != 1:
-              raise SystemExit("Expected exactly one ArchiveBox wheel and one sdist")
-          artifacts = [root / "COMMIT_SHA", wheels[0], sdists[0]]
-          checksum_lines = []
-          for artifact in artifacts:
-              digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
-              line = f"{digest}  {artifact.name}"
-              checksum_lines.append(line)
-              print(line)
-          Path("dist/SHA256SUMS").write_text("\n".join(checksum_lines) + "\n")
-          PY
+      - name: UV install archivebox and archivebox/pkgs/* locally for tests
+        run: uv sync --all-extras --no-sources --no-cache
 
-      - name: Release wheel import and CLI smoke
-        if: ${{ !inputs.full_tests }}
-        shell: bash
+      - name: UV run archivebox init + archivebox version
         run: |
-          set -Eeuo pipefail
-          shopt -s nullglob
-          wheels=(dist/archivebox-*.whl)
-          [[ "${#wheels[@]}" -eq 1 ]]
-          uv sync --locked --no-cache --no-sources --no-dev --no-install-project
-          uv pip install --no-cache --no-deps --python .venv/bin/python "${wheels[0]}"
-          unset PYTHONPATH
-          cd "$(mktemp -d)"
-          VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project python -c 'import archivebox'
-          VIRTUAL_ENV="$GITHUB_WORKSPACE/.venv" uv run --no-cache --active --no-project archivebox version
-
-      - name: Upload every distribution
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: python-distributions
-          path: dist/*
-          if-no-files-found: error
-          retention-days: 7
-
-  install-smoke:
-    name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }}
-    if: inputs.full_tests
-    needs: build
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ${{ fromJSON(inputs.platforms) }}
-        python: ["3.13", "3.14"]
-        artifact: [wheel, sdist]
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: ${{ env.UV_VERSION }}
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Download distributions
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: python-distributions
-          path: ${{ runner.temp }}/python-distributions
-
-      - name: Install and smoke-test the built ${{ matrix.artifact }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          shopt -s nullglob
-
-          case "${{ matrix.artifact }}" in
-            wheel) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.whl) ;;
-            sdist) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.tar.gz) ;;
-            *) exit 2 ;;
-          esac
-          [[ "${#candidates[@]}" -eq 1 ]]
-          artifact="${candidates[0]}"
-
-          smoke_root="$(mktemp -d "$RUNNER_TEMP/archivebox-package-smoke.XXXXXX")"
-          trap 'rm -rf "$smoke_root"' EXIT
-          smoke_env="$smoke_root/env"
-          data_dir="$smoke_root/data"
-          mkdir -p "$data_dir"
-
-          dependency_artifacts=()
-          while IFS= read -r dependency_artifact; do
-            dependency_artifacts+=("$dependency_artifact")
-          done < <(uv run --no-cache --no-project python - <<'PY'
-          import tomllib
-
-          locked = {
-              package["name"]: package
-              for package in tomllib.load(open("uv.lock", "rb"))["package"]
-          }
-          for package_name in ("abx-dl", "abx-plugins", "abxpkg", "abxbus"):
-              package = locked[package_name]
-              wheels = package.get("wheels") or []
-              if len(wheels) != 1:
-                  raise SystemExit(f"Expected exactly one locked wheel for {package_name}")
-              print(wheels[0]["url"])
-          PY
-          )
-
-          uv venv --python "${{ matrix.python }}" "$smoke_env"
-          uv pip install --no-cache \
-            --python "$smoke_env" \
-            "${dependency_artifacts[@]}" \
-            "$artifact"
-
-          unset PYTHONPATH
-          cd "$data_dir"
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync python - <<'PY'
-          import os
-          from pathlib import Path
-
-          import archivebox
-
-          installed_path = Path(archivebox.__file__).resolve()
-          environment_path = Path(os.environ["VIRTUAL_ENV"]).resolve()
-          workspace_path = Path(os.environ["GITHUB_WORKSPACE"]).resolve()
-          assert installed_path.is_relative_to(environment_path), (installed_path, environment_path)
-          assert not installed_path.is_relative_to(workspace_path), (installed_path, workspace_path)
-          print(installed_path)
-          PY
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox version
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox init
-          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox status
-
-          if [[ "${{ runner.os }}" == "Linux" ]]; then
-            root_tool_dir="/root/archivebox-package-smoke-${{ matrix.artifact }}-${{ matrix.python }}"
-            uv_binary="$(command -v uv)"
-            sudo rm -rf "$root_tool_dir"
-            sudo mkdir -p "$root_tool_dir/data"
-
-            root_install_args=()
-            for dependency_artifact in "${dependency_artifacts[@]}"; do
-              root_install_args+=(--with "$dependency_artifact")
-            done
-            sudo UV_TOOL_DIR="$root_tool_dir/tools" UV_TOOL_BIN_DIR="$root_tool_dir/bin" \
-              "$uv_binary" tool install --no-cache --python "${{ matrix.python }}" \
-              "${root_install_args[@]}" "$artifact"
-
-            sudo useradd --system --create-home --home-dir /var/lib/archivebox --shell /bin/bash archivebox 2>/dev/null || true
-            sudo rm -rf /var/lib/archivebox
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data' && archivebox init"
-            sudo stat -c %U /var/lib/archivebox | grep -qx archivebox
-            sudo stat -c %U "$root_tool_dir/data" | grep -qx archivebox
-            if sudo find "$root_tool_dir/data" -xdev -user root -print -quit | grep -q .; then
-              echo "Root-owned files remain in $root_tool_dir/data" >&2
-              exit 1
-            fi
-
-            sudo mkdir "$root_tool_dir/external"
-            sudo ln -s "$root_tool_dir/external" "$root_tool_dir/data/external-link"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data' && archivebox status"
-            sudo stat -c %U "$root_tool_dir/external" | grep -qx root
-
-            sudo mkdir "$root_tool_dir/data-universal-init"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-universal-init' && archivebox status --init"
-            sudo stat -c %U "$root_tool_dir/data-universal-init" | grep -qx archivebox
-
-            sudo mkdir "$root_tool_dir/data-install"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-install' && archivebox install --dry-run"
-            sudo stat -c %U "$root_tool_dir/data-install" | grep -qx archivebox
-
-            sudo useradd --create-home archivebox-smoke-user
-            smoke_user_group="$(id -gn archivebox-smoke-user)"
-            sudo mkdir "$root_tool_dir/data-mixed-owner"
-            sudo chown archivebox-smoke-user:root "$root_tool_dir/data-mixed-owner"
-            sudo env PATH="$root_tool_dir/bin:$PATH" \
-              bash -c "cd '$root_tool_dir/data-mixed-owner' && archivebox init"
-            sudo stat -c %U:%G "$root_tool_dir/data-mixed-owner" | grep -qx "archivebox-smoke-user:$smoke_user_group"
-            if sudo find "$root_tool_dir/data-mixed-owner" -xdev -user root -print -quit | grep -q .; then
-              echo "Root-owned files remain in $root_tool_dir/data-mixed-owner" >&2
-              exit 1
-            fi
-            sudo rm -rf "$root_tool_dir"
-          fi
+          mkdir -p data && cd data
+          uv run --no-sync --no-sources archivebox init \
+          && uv run --no-sync --no-sources archivebox version
+          # && uv run archivebox add 'https://example.com' \
+          # && uv run archivebox status \
+          # || (echo "UV Failed to run archivebox!" && exit 1)

.github/workflows/prepare-release.yml

@@ -1,55 +0,0 @@
-name: Prepare release version
-
-on:
-  workflow_call:
-    outputs:
-      run_jobs:
-        description: Whether this revision owns its release version and should run expensive jobs
-        value: ${{ jobs.prepare.outputs.run_jobs }}
-
-permissions:
-  contents: write
-
-jobs:
-  prepare:
-    name: Prepare the version before starting expensive jobs
-    permissions:
-      contents: write
-    runs-on: ubuntu-24.04
-    outputs:
-      run_jobs: ${{ github.event_name != 'push' || steps.channel.outputs.stable == 'true' || steps.version.outputs.action == 'ready' || steps.version.outputs.action == 'reserved' }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          token: ${{ github.event_name == 'push' && secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - name: Keep synchronized stable branches on the released version
-        id: channel
-        if: github.event_name == 'push'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
-          git fetch --no-tags origin main
-          MAIN_SHA="$(git rev-parse origin/main)"
-          # Stable reconciliation may preserve dev history with a merge commit.
-          # Its tree still matches main; bumping it would undo synchronization
-          # before the stable publisher can advance the Docker :dev tag.
-          if [[ "$GITHUB_REF_NAME" == dev && "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
-             git merge-base --is-ancestor "$MAIN_SHA" "$GITHUB_SHA" &&
-             git diff --quiet "$MAIN_SHA" "$GITHUB_SHA"; then
-            echo 'stable=true' >> "$GITHUB_OUTPUT"
-          else
-            echo 'stable=false' >> "$GITHUB_OUTPUT"
-          fi
-
-      - id: version
-        if: github.event_name == 'push' && steps.channel.outputs.stable != 'true'
-        uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
-        with:
-          package: archivebox
-          branch: ${{ github.ref_name }}
-          tag-prefix: v
-          version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}

.github/workflows/release-runner.yml

@@ -0,0 +1,45 @@
+name: Release State
+
+on:
+  push:
+    branches:
+      - '**'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  id-token: write
+
+jobs:
+  release-state:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+          ref: ${{ github.ref_name }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: false
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Configure git identity
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Run release script
+        env:
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+          GH_TOKEN: ${{ github.token }}
+          PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
+        run: ./bin/release.sh

.github/workflows/release.yml

@@ -1,595 +1,43 @@
 name: Release
 
+# Orchestrates the full release pipeline:
+#   1. Build and publish pip package to PyPI
+#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
+#   3. Build Docker images (in parallel with deb/brew)
+#
+# Individual workflows also run on push for CI (see their own triggers).
+# Manual fallback for the hosted release pipeline. The normal prerelease loop
+# is run by bin/release.sh locally so published GitHub releases do not launch a
+# duplicate release job.
+
 on:
-  workflow_run:
-    workflows: [CI]
-    types: [completed]
-    branches: [dev, main]
   workflow_dispatch:
-    inputs:
-      release_branch:
-        description: Branch being promoted
-        required: true
-        type: choice
-        options: [main, dev]
-      release_sha:
-        description: Exact tested commit to release
-        required: true
-        type: string
-      ci_run_id:
-        description: Successful CI run containing the tested Python artifacts
-        required: true
-        type: string
-      docker_digest_run_id:
-        description: Manual Docker digest build run containing the tested image digests
-        required: true
-        type: string
 
-permissions: {}
-
-concurrency:
-  group: release-${{ github.repository }}-${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
-  cancel-in-progress: false
-
-env:
-  DOCKERHUB_IMAGE: archivebox/archivebox
-  GHCR_IMAGE: ghcr.io/archivebox/archivebox
-  RELEASE_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.release_branch || github.event.workflow_run.head_branch }}
-  RELEASE_SHA: ${{ github.event_name == 'workflow_dispatch' && inputs.release_sha || github.event.workflow_run.head_sha }}
+permissions:
+  contents: write
+  packages: write
+  id-token: write
 
 jobs:
-  candidate:
-    if: >-
-      github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.event == 'push' &&
-      (github.event.workflow_run.head_branch == 'dev' || github.event.workflow_run.head_branch == 'main') ||
-      (github.event_name == 'workflow_dispatch' && inputs.release_branch == 'main')
-    runs-on: ubuntu-24.04
-    permissions:
-      actions: read
-      contents: read
-    outputs:
-      ready: ${{ steps.verified.outputs.ready }}
-      artifact_run_id: ${{ steps.verified.outputs.artifact_run_id }}
-      digest_run_id: ${{ steps.verified.outputs.digest_run_id }}
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          ref: ${{ env.RELEASE_SHA }}
-      - id: owner
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          BRANCH_SHA="$(git ls-remote origin "refs/heads/$RELEASE_BRANCH" | cut -f1)"
-          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && "$BRANCH_SHA" != "$RELEASE_SHA" ]]; then
-            echo 'ready=false' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          VERSION="$(sed -nE 's/^version = "([^"]+)".*/\1/p' pyproject.toml | head -n 1)"
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
-              echo "Refusing to publish non-stable version $VERSION as a main release" >&2
-              exit 1
-            }
-            git fetch --no-tags origin main
-            git merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
-              echo "Refusing to publish $RELEASE_SHA: it is not on main" >&2
-              exit 1
-            }
-            TAG_TARGET="$(git ls-remote origin "refs/tags/v$VERSION" | cut -f1)"
-            if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch && -n "$TAG_TARGET" && "$TAG_TARGET" != "$RELEASE_SHA" ]]; then
-              echo 'ready=false' >> "$GITHUB_OUTPUT"
-              exit 0
-            fi
-            echo 'ready=true' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          # A stable release fast-forwards dev to main. Only main publishes it;
-          # the resulting dev workflow is not a new release candidate.
-          if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]]; then
-            echo 'ready=false' >> "$GITHUB_OUTPUT"
-            exit 0
-          fi
-          TARGET="$(git ls-remote origin "refs/tags/release-candidate/$VERSION" | awk '{print $1}')"
-          [[ "$TARGET" == "$RELEASE_SHA" ]] && echo 'ready=true' >> "$GITHUB_OUTPUT" || echo 'ready=false' >> "$GITHUB_OUTPUT"
+  pip:
+    name: Publish to PyPI
+    uses: ./.github/workflows/pip.yml
+    secrets: inherit
 
-      - name: Require successful full CI for this commit
-        id: verified
-        if: steps.owner.outputs.ready == 'true'
-        env:
-          GH_TOKEN: ${{ github.token }}
-          MANUAL_CI_RUN_ID: ${{ inputs.ci_run_id }}
-          MANUAL_DIGEST_RUN_ID: ${{ inputs.docker_digest_run_id }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then
-            CI_RUN_ID="$MANUAL_CI_RUN_ID"
-            DIGEST_RUN_ID="$MANUAL_DIGEST_RUN_ID"
-            CI_STATUS="$(gh run view "$CI_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status,conclusion,headSha,event,name \
-              --jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
-            [[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
-          else
-            CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
-              --commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
-              --json databaseId,status,conclusion \
-              --jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
-            [[ -n "$CI_RUN_ID" ]] || exit 0
-            DIGEST_RUN_ID="$CI_RUN_ID"
-          fi
-          ARTIFACT_RUN_ID="$CI_RUN_ID"
-          {
-            echo 'ready=true'
-            echo "artifact_run_id=$ARTIFACT_RUN_ID"
-            echo "digest_run_id=$DIGEST_RUN_ID"
-          } >> "$GITHUB_OUTPUT"
+  debian:
+    name: Build .deb packages
+    needs: pip
+    uses: ./.github/workflows/debian.yml
+    secrets: inherit
 
-  python-release:
-    name: Publish tested Python artifacts
-    needs: candidate
-    if: needs.candidate.outputs.ready == 'true'
-    runs-on: ubuntu-24.04
-    env:
-      CI_RUN_ID: ${{ needs.candidate.outputs.artifact_run_id }}
-      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
-    environment: pypi
-    permissions:
-      actions: read
-      contents: write
-      deployments: read
-      id-token: write
-    outputs:
-      version: ${{ steps.version.outputs.version }}
-      tag: ${{ steps.version.outputs.tag }}
-      release_ready: ${{ steps.release-state.outputs.ready }}
-    steps:
-      - name: Checkout the tested commit
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          ref: ${{ env.RELEASE_SHA }}
-          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
+  homebrew:
+    name: Update Homebrew formula
+    needs: pip
+    uses: ./.github/workflows/homebrew.yml
+    secrets: inherit
 
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      # This precedes every stable upload, including workflow_dispatch. A green
-      # build is not evidence that our automatically updated test servers ran it.
-      - name: Download tested Docker base-image metadata
-        if: env.RELEASE_BRANCH == 'main'
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          path: ${{ runner.temp }}/tested-docker-artifacts
-          pattern: digest-*
-          merge-multiple: true
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
-
-      - name: Require matching acceptance on Cabbage
-        if: env.RELEASE_BRANCH == 'main'
-        env:
-          GH_TOKEN: ${{ github.token }}
-          STAGING_DOCKER_ARTIFACT_DIR: ${{ runner.temp }}/tested-docker-artifacts
-        run: uv run --no-project python bin/verify-staging-release.py
-
-      - name: Resolve release binaries through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" >/dev/null
-          for name in uv gh git jq curl; do
-            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
-            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
-          done
-          {
-            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
-            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
-            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-            echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
-          } >> "$GITHUB_ENV"
-
-      - name: Download exact tested distributions
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: python-distributions
-          path: ${{ runner.temp }}/python-distributions
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.CI_RUN_ID }}
-
-      - name: Record release version
-        id: version
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$($UV_BINARY run --no-cache --no-project python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
-          echo "tag=v$VERSION" >> "$GITHUB_OUTPUT"
-
-      - name: Validate release branch and version
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION='${{ steps.version.outputs.version }}'
-          case "$RELEASE_BRANCH" in
-            main)
-              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
-                echo "Refusing to publish non-stable version $VERSION from main" >&2
-                exit 1
-              }
-              ;;
-            dev)
-              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]] || {
-                echo "Refusing to publish ArchiveBox $VERSION from dev; only archivebox:main may publish non-prerelease versions." >&2
-                exit 1
-              }
-              ;;
-            *)
-              echo "Refusing to publish ArchiveBox $VERSION from unsupported branch $RELEASE_BRANCH; only archivebox:dev and archivebox:main may publish." >&2
-              exit 1
-              ;;
-          esac
-
-      - name: Publish the exact tested distributions
-        env:
-          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-          RELEASE_DISTRIBUTIONS_DIR: ${{ runner.temp }}/python-distributions
-        run: ./bin/release.sh
-
-      - name: Confirm this tested commit owns the release tag
-        id: release-state
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          TARGET=""
-          while read -r sha _; do
-            TARGET="$sha"
-          done < <("$GIT_BINARY" ls-remote origin "refs/tags/${{ steps.version.outputs.tag }}")
-          if [[ "$TARGET" == "$RELEASE_SHA" ]]; then
-            echo "ready=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "ready=false" >> "$GITHUB_OUTPUT"
-          fi
-
-  docker-release:
-    name: Publish tested multiarch image
-    if: >-
-      (github.event_name == 'workflow_dispatch') ||
-      (github.event.workflow_run.conclusion == 'success' &&
-       github.event.workflow_run.event == 'push')
-    needs: [candidate, python-release]
-    runs-on: ubuntu-24.04
-    env:
-      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
-    permissions:
-      actions: read
-      contents: write
-      packages: write
-    steps:
-      - name: Checkout the tested commit
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 0
-          ref: ${{ env.RELEASE_SHA }}
-          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
-
-      - name: Reconcile dev to the tested stable release
-        if: env.RELEASE_BRANCH == 'main' && needs.python-release.outputs.release_ready == 'true'
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          git fetch --no-tags origin main dev
-          MAIN_SHA="$(git rev-parse origin/main)"
-          DEV_SHA="$(git rev-parse origin/dev)"
-          if [[ "$MAIN_SHA" != "$RELEASE_SHA" || "$DEV_SHA" == "$RELEASE_SHA" ]]; then
-            exit 0
-          fi
-          push_dev() {
-            if ! git push origin "$1:refs/heads/dev"; then
-              CURRENT_DEV="$(git ls-remote origin refs/heads/dev | cut -f1)"
-              if [[ "$CURRENT_DEV" == "$DEV_SHA" ]]; then
-                echo "Failed to push dev although it still points to $DEV_SHA" >&2
-                exit 1
-              fi
-              echo "Leaving dev at $CURRENT_DEV: it advanced during release reconciliation"
-            fi
-          }
-          if git merge-base --is-ancestor "$DEV_SHA" "$RELEASE_SHA"; then
-            push_dev "$RELEASE_SHA"
-            exit 0
-          else
-            STATUS=$?
-            [[ "$STATUS" -eq 1 ]] || exit "$STATUS"
-          fi
-          BASE_SHA="$(git merge-base "$MAIN_SHA" "$DEV_SHA")"
-          if git diff --quiet "$BASE_SHA" "$DEV_SHA" -- . ':!pyproject.toml' ':!uv.lock' ':!etc/package.json'; then
-            :
-          else
-            STATUS=$?
-            [[ "$STATUS" -eq 1 ]] || exit "$STATUS"
-            echo "Leaving dev at $DEV_SHA: it contains changes beyond release metadata"
-            exit 0
-          fi
-          normalized_metadata() {
-            case "$2" in
-              pyproject.toml)
-                git show "$1:$2" | awk '
-                  /^\[/ {section = $0}
-                  section == "[project]" && /^version = "[^"]+"$/ {print "version = \"__VERSION__\""; next}
-                  section == "[tool.bumpver]" && /^current_version = "[^"]+"$/ {print "current_version = \"__VERSION__\""; next}
-                  {print}
-                ' ;;
-              uv.lock)
-                git show "$1:$2" | awk 'previous == "name = \"archivebox\"" && /^version = / {print "version = \"__VERSION__\""; previous = $0; next} {print; previous = $0}' ;;
-              etc/package.json)
-                git show "$1:$2" | sed -E 's/^(  "version": )"[^"]+"(,?)$/\1"__VERSION__"\2/' ;;
-            esac
-          }
-          for path in pyproject.toml uv.lock etc/package.json; do
-            BASE_METADATA="$(normalized_metadata "$BASE_SHA" "$path")"
-            DEV_METADATA="$(normalized_metadata "$DEV_SHA" "$path")"
-            if [[ "$BASE_METADATA" != "$DEV_METADATA" ]]; then
-              echo "Leaving dev at $DEV_SHA: $path contains changes beyond release version metadata"
-              exit 0
-            fi
-          done
-          # A concurrent release bot may bump only dev's own version metadata.
-          # Keep that commit as a parent while preserving the exact tested stable tree.
-          git switch --detach "$MAIN_SHA"
-          git -c user.name='github-actions[bot]' -c user.email='41898282+github-actions[bot]@users.noreply.github.com' \
-            merge -s ours --no-ff -m "Merge tested stable release into dev" "$DEV_SHA"
-          MERGE_SHA="$(git rev-parse HEAD)"
-          push_dev "$MERGE_SHA"
-
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-
-      - name: Resolve release and Docker binaries through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
-          for name in uv gh git jq docker; do
-            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
-            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
-          done
-          {
-            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
-            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
-            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-            echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
-          } >> "$GITHUB_ENV"
-
-      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
-        with:
-          # These images were already built and tested; only publish manifests.
-          driver: docker
-
-      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ github.token }}
-
-      - name: Download exact tested image digests
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          path: /tmp/digests
-          pattern: digest-*
-          merge-multiple: true
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
-
-      - name: Verify exact successful digest build run
-        shell: bash
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          set -Eeuo pipefail
-          RUN_STATUS="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json status --jq '.status')"
-          RUN_CONCLUSION="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json conclusion --jq '.conclusion')"
-          [[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]]
-          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
-            RUN_SHA="$($GH_BINARY run view "$DOCKER_DIGEST_RUN_ID" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha')"
-            [[ "$RUN_SHA" == "$RELEASE_SHA" ]]
-          fi
-
-      - name: Verify digest set and collect tags
-        id: docker_meta
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          [[ "${#DIGESTS[@]}" -eq 2 ]]
-          for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
-          [[ "$(< /tmp/digests/source-ref)" == "$RELEASE_SHA" ]]
-
-          VERSION='${{ needs.python-release.outputs.version }}'
-          SHORT_SHA="${RELEASE_SHA::12}"
-          TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}^{}")"
-          TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
-          if [[ -z "$TAG_TARGET" ]]; then
-            TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
-            TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
-          fi
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || {
-              echo "Refusing to tag :latest with prerelease $VERSION" >&2
-              exit 1
-            }
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] || {
-              echo "Refusing to tag :latest: v$VERSION does not point to $RELEASE_SHA" >&2
-              exit 1
-            }
-            $GIT_BINARY fetch --no-tags origin main
-            $GIT_BINARY merge-base --is-ancestor "$RELEASE_SHA" origin/main || {
-              echo "Refusing to tag :latest: $RELEASE_SHA is not on main" >&2
-              exit 1
-            }
-            MAIN_VERSION="$($GIT_BINARY show origin/main:pyproject.toml | sed -nE 's/^version = "([^"]+)".*/\1/p' | head -n 1)"
-            [[ "$MAIN_VERSION" == "$VERSION" ]] || {
-              echo "Refusing to tag :latest: main is now $MAIN_VERSION, not $VERSION" >&2
-              exit 1
-            }
-          fi
-          SYNC_DEV=false
-          if [[ "$RELEASE_BRANCH" == main ]]; then
-            $GIT_BINARY fetch --no-tags origin main dev
-            MAIN_TARGET="$($GIT_BINARY rev-parse origin/main)"
-            DEV_TARGET="$($GIT_BINARY rev-parse origin/dev)"
-            # Recovery can add CI/docs fixes after the immutable release tag.
-            # Both branches and every shipped application file must agree.
-            if $GIT_BINARY merge-base --is-ancestor "$MAIN_TARGET" "$DEV_TARGET" &&
-               $GIT_BINARY diff --quiet "$MAIN_TARGET" "$DEV_TARGET" &&
-               $GIT_BINARY diff --quiet "$RELEASE_SHA" "$MAIN_TARGET" -- . ':!.github' ':!README.md'; then
-              SYNC_DEV=true
-            fi
-          fi
-          {
-            echo 'dockerhub_tags<<EOF'
-            if [[ "$RELEASE_BRANCH" == dev ]]; then
-              echo "${DOCKERHUB_IMAGE}:dev"
-            elif [[ "$RELEASE_BRANCH" == main ]]; then
-              echo "${DOCKERHUB_IMAGE}:latest"
-              [[ "$SYNC_DEV" == true ]] && echo "${DOCKERHUB_IMAGE}:dev"
-            fi
-            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
-            echo 'EOF'
-            echo 'ghcr_tags<<EOF'
-            if [[ "$RELEASE_BRANCH" == dev ]]; then
-              echo "${GHCR_IMAGE}:dev"
-            elif [[ "$RELEASE_BRANCH" == main ]]; then
-              echo "${GHCR_IMAGE}:latest"
-              [[ "$SYNC_DEV" == true ]] && echo "${GHCR_IMAGE}:dev"
-            fi
-            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
-            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
-            echo 'EOF'
-          } >> "$GITHUB_OUTPUT"
-
-      - name: Create Docker Hub manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          TAG_ARGS=()
-          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
-          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Create GHCR manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          TAG_ARGS=()
-          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
-          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Verify published tags contain the exact tested images
-        env:
-          DOCKERHUB_TAGS: ${{ steps.docker_meta.outputs.dockerhub_tags }}
-          GHCR_TAGS: ${{ steps.docker_meta.outputs.ghcr_tags }}
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          # Both architectures already passed CLI/version/capture acceptance by
-          # digest. Verify promotion preserves those exact images and attestations
-          # instead of downloading and running the same image again on one arch.
-          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
-          EXPECTED="$("$DOCKER_BINARY" buildx imagetools create --dry-run "${REFS[@]}" | "$JQ_BINARY" -ceS '.manifests | sort_by(.digest)')"
-          "$JQ_BINARY" -e 'map(select(.platform.os == "linux") | .platform.architecture) | sort == ["amd64", "arm64"]' <<< "$EXPECTED"
-          while IFS= read -r image; do
-            [[ -n "$image" ]] || continue
-            ACTUAL="$("$DOCKER_BINARY" buildx imagetools inspect --raw "$image" | "$JQ_BINARY" -ceS 'select(.schemaVersion == 2) | .manifests | sort_by(.digest)')"
-            [[ "$ACTUAL" == "$EXPECTED" ]] || { echo "Published tag differs from tested images: $image" >&2; exit 1; }
-            echo "Verified exact tested images: $image"
-          done < <(printf '%s\n' "$DOCKERHUB_TAGS" "$GHCR_TAGS")
-
-      - name: Record published image source
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          # workflow_run can report the default branch's head even when this
-          # run published a different branch/SHA. Preserve the verified target.
-          [[ "$RELEASE_BRANCH" == dev || "$RELEASE_BRANCH" == main ]]
-          [[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]
-          mkdir -p "$RUNNER_TEMP/published-image-source"
-          # shellcheck disable=SC2016
-          "$JQ_BINARY" -n --arg branch "$RELEASE_BRANCH" --arg sha "$RELEASE_SHA" \
-            '{branch: $branch, sha: $sha}' > "$RUNNER_TEMP/published-image-source/source.json"
-
-      - name: Upload published image source
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: published-image-source
-          path: ${{ runner.temp }}/published-image-source/source.json
-          if-no-files-found: error
-
-      - name: Update Docker Hub README
-        uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: archivebox/archivebox
-
-  cascade:
-    name: Cascade verified release
-    if: needs.python-release.outputs.release_ready == 'true'
-    needs:
-      - candidate
-      - python-release
-      - docker-release
-    permissions:
-      actions: read
-      contents: read
-    uses: ArchiveBox/monorepo/.github/workflows/cascade-release.yml@main
-    with:
-      upstream_package: archivebox
-      upstream_version: ${{ needs.python-release.outputs.version }}
-      upstream_artifact_name: python-distributions
-      upstream_run_id: ${{ needs.candidate.outputs.artifact_run_id }}
-    secrets:
-      release_token: ${{ secrets.RELEASE_GH_TOKEN }}
+  docker:
+    name: Build Docker images
+    needs: pip
+    uses: ./.github/workflows/docker.yml
+    secrets: inherit

.github/workflows/screenshots.yml

@@ -1,81 +0,0 @@
-name: Capture publicsite screenshots
-
-on:
-  schedule:
-    - cron: '37 5,17 * * *'
-  workflow_dispatch:
-
-permissions:
-  contents: read
-  actions: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: false
-
-jobs:
-  changes:
-    runs-on: ubuntu-24.04
-    outputs:
-      changed: ${{ steps.check.outputs.changed }}
-      key: ${{ steps.check.outputs.key }}
-    steps:
-      - id: check
-        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
-
-  deploy:
-    needs: changes
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          fetch-depth: 1
-
-      - name: Setup Python and uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: false
-
-      - name: Install ArchiveBox and screenshot dependencies
-        run: uv sync --no-cache --frozen
-
-      - name: Build UI screenshot gallery
-        run: bin/collect_ui_screenshots.sh
-        env:
-          BASE_URL: http://archivebox.localhost:5797
-
-      - name: Verify gallery was generated from this exact revision
-        run: |
-          uv run --no-cache --no-sync python - <<'PY'
-          import json
-          import os
-          import re
-          import tomllib
-          from pathlib import Path
-
-          provenance = json.loads(Path("publicsite/screenshots/build.json").read_text())
-          version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
-          assert provenance["revision"] == os.environ["GITHUB_SHA"], provenance
-          assert provenance["version"] == version, provenance
-          assert provenance["capture_count"] >= 3 * 35, provenance
-          assert len(provenance["files"]) == provenance["capture_count"], provenance
-          assert all(not re.match(r"^\d+-", name) for name in provenance["files"]), provenance
-          PY
-
-      - name: Save validated screenshots
-        uses: actions/upload-artifact@v4
-        with:
-          name: site-screenshots
-          path: publicsite/screenshots
-          if-no-files-found: error
-          retention-days: 90
-
-  complete:
-    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
-    needs: [changes, deploy]
-    if: needs.changes.outputs.changed == 'true'
-    runs-on: ubuntu-24.04
-    steps:
-      - run: echo 'Scheduled work completed successfully.'

.github/workflows/staging-acceptance.yml

@@ -1,117 +0,0 @@
-name: Staging acceptance
-
-on:
-  workflow_run:
-    workflows: [Release]
-    types: [completed]
-
-permissions: {}
-
-concurrency:
-  group: staging-acceptance
-  cancel-in-progress: false
-
-jobs:
-  published:
-    if: github.event.workflow_run.conclusion == 'success'
-    runs-on: ubuntu-24.04
-    permissions:
-      actions: read
-    outputs:
-      ready: ${{ steps.source.outputs.ready }}
-      sha: ${{ steps.source.outputs.sha }}
-    steps:
-      - id: image
-        env:
-          GH_TOKEN: ${{ github.token }}
-          RELEASE_RUN_ID: ${{ github.event.workflow_run.id }}
-        run: |
-          set -Eeuo pipefail
-          published=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RELEASE_RUN_ID/jobs?per_page=100" --jq 'any(.jobs[]; .name == "Publish tested multiarch image" and .conclusion == "success")')
-          echo "published=$published" >> "$GITHUB_OUTPUT"
-      - name: Download published image source
-        if: steps.image.outputs.published == 'true'
-        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: published-image-source
-          path: ${{ runner.temp }}/published-image-source
-          github-token: ${{ github.token }}
-          repository: ${{ github.repository }}
-          run-id: ${{ github.event.workflow_run.id }}
-      - name: Select published dev image
-        id: source
-        if: steps.image.outputs.published == 'true'
-        run: |
-          set -Eeuo pipefail
-          # The triggering run may report the default branch as its head;
-          # this artifact names the image source verified by the publisher.
-          source="$RUNNER_TEMP/published-image-source/source.json"
-          jq -e 'type == "object" and (keys | sort) == ["branch", "sha"]' "$source" >/dev/null
-          branch=$(jq -er '.branch | select(type == "string")' "$source")
-          sha=$(jq -er '.sha | select(type == "string")' "$source")
-          [[ "$branch" == dev || "$branch" == main ]]
-          [[ "$sha" =~ ^[0-9a-f]{40}$ ]]
-          echo "sha=$sha" >> "$GITHUB_OUTPUT"
-          if [[ "$branch" == dev ]]; then
-            echo 'ready=true' >> "$GITHUB_OUTPUT"
-          else
-            echo 'ready=false' >> "$GITHUB_OUTPUT"
-          fi
-  acceptance:
-    # The privileged key is never exposed to PRs or arbitrary dispatch refs.
-    needs: published
-    if: needs.published.outputs.ready == 'true'
-    runs-on: [self-hosted, Linux, X64, ugnas-linux]
-    environment: staging
-    timeout-minutes: 110
-    permissions:
-      contents: read
-      deployments: write
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [cabbage]
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
-        with:
-          ref: ${{ needs.published.outputs.sha }}
-          fetch-depth: 0
-          persist-credentials: false
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e
-        with:
-          version: '0.11.3'
-      - name: Prepare browser and restricted staging identity
-        env:
-          STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
-          STAGING_KNOWN_HOSTS: ${{ secrets.STAGING_KNOWN_HOSTS }}
-          ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/staging-browser
-        run: |
-          set -Eeuo pipefail
-          umask 077
-          printf '%s\n' "$STAGING_SSH_KEY" > "$RUNNER_TEMP/staging-key"
-          printf '%s\n' "$STAGING_KNOWN_HOSTS" > "$RUNNER_TEMP/staging-known-hosts"
-          # Use the candidate's normal provider/cache path, not a second
-          # Playwright-managed browser that bypasses abxpkg validation.
-          ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
-          CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
-          uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
-          CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
-          test -x "$CHROME_BINARY"
-          echo "CHROME_BINARY=$CHROME_BINARY" >> "$GITHUB_ENV"
-      - name: Verify actual deployment, full captures and rendered snapshots
-        env:
-          GH_TOKEN: ${{ github.token }}
-          STAGING_TARGET: ${{ matrix.target }}
-          STAGING_SHA: ${{ needs.published.outputs.sha }}
-          STAGING_KEY_FILE: ${{ runner.temp }}/staging-key
-          STAGING_KNOWN_HOSTS_FILE: ${{ runner.temp }}/staging-known-hosts
-        run: uv run --no-project --with playwright==1.63.0 python bin/check-staging-deployment.py
-      - name: Remove staging identity
-        if: always()
-        run: rm -f "$RUNNER_TEMP/staging-key" "$RUNNER_TEMP/staging-known-hosts"
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
-        if: always()
-        with:
-          name: staging-${{ matrix.target }}-${{ needs.published.outputs.sha }}
-          path: staging-evidence/
-          if-no-files-found: error

.github/workflows/test-parallel.yml

@@ -2,254 +2,295 @@ name: Parallel Tests
 
 on:
   workflow_dispatch:
-    inputs:
-      test_file:
-        description: Run one discovered test file
-        required: true
-        default: archivebox/tests/test_cli_list.py
-        type: string
-  workflow_call:
-    inputs:
-      test_file:
-        description: Run one discovered test file; empty runs the complete matrix
-        required: false
-        default: ""
-        type: string
+  pull_request:
+    branches: [dev, main, master]
+  push:
+    branches: [dev]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
   UV_NO_SOURCES: "1"
-  CI_PYTHON_VERSION: "3.13"
 
 jobs:
   discover-tests:
-    name: Discover every test
-    # Selection must run on GitHub so an offline NAS cannot block the fallback.
+    name: Discover test files
     runs-on: ubuntu-24.04
     outputs:
-      tests: ${{ steps.discover.outputs.tests }}
-      runner: ${{ steps.runner.outputs.use-runner || '["ubuntu-24.04"]' }}
+      test-files: ${{ steps.set-matrix.outputs.test-files }}
+      plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}
 
     steps:
-      - name: Select available CI capacity
-        id: runner
-        if: vars.UGNAS_CI_ENABLED == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
-        uses: ArchiveBox/monorepo/.github/actions/select-ci-runner@main
-        with:
-          github-token: ${{ secrets.RUNNER_STATUS_TOKEN }}
-
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
-        with:
-          python-version: ${{ env.CI_PYTHON_VERSION }}
-
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
-        with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
-
-      - name: Resolve discovery tools through abxpkg
-        env:
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+      - name: Discover test files
+        id: set-matrix
         run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          test -n "$ABXPKG_SPEC"
-          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
-          export PYTHON_BINARY="$pythonLocation/bin/python"
-          test -x "$PYTHON_BINARY"
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-            --install \
-            --lib="$ABXPKG_LIB_DIR" \
-            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
-          {
-            echo "PYTHON_BINARY=$ABXPKG_LIB_DIR/env/bin/python"
-            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
-          } >> "$GITHUB_ENV"
-          test -L "$ABXPKG_LIB_DIR/env/bin/python"
-          test -x "$ABXPKG_LIB_DIR/env/bin/python"
-          test -L "$ABXPKG_LIB_DIR/env/bin/jq"
-          test -x "$ABXPKG_LIB_DIR/env/bin/jq"
+          # Find all main test files
+          all_tests=$(find archivebox/tests -maxdepth 1 -name "test_*.py" -type f | sort)
 
-      - name: Discover every test exactly once
-        id: discover
-        env:
-          TEST_FILE: ${{ inputs.test_file }}
-          UGNAS_CI_MAX_JOBS: ${{ vars.UGNAS_CI_MAX_JOBS || 3 }}
+          # Create JSON array with test file info
+          json_array="["
+          first=true
+          for test_file in $all_tests; do
+            if [ "$first" = true ]; then
+              first=false
+            else
+              json_array+=","
+            fi
+
+            # Extract a display name for the test
+            name="main/$(basename $test_file .py | sed 's/^test_//')"
+
+            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
+          done
+          json_array+="]"
+
+          echo "test-files=$json_array" >> $GITHUB_OUTPUT
+          echo "Found $(echo $all_tests | wc -w) test files"
+          echo "$json_array" | jq '.'
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Discover plugin tests
+        id: set-plugin-matrix
         run: |
-          set -Eeuo pipefail
-          mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
-          test "${#output[@]}" -eq 2
-          if [[ -n "$TEST_FILE" ]]; then
-            output[1]="$("$JQ_BINARY" -ce '[.[] | select(.paths == [env.TEST_FILE])] | select(length == 1)' <<< "${output[1]}")"
-          fi
-          echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
-          echo "${output[0]}"
-          "$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"
+          plugin_tests=$(find abx-plugins/abx_plugins/plugins -maxdepth 2 -type d -name tests | sed 's#abx-plugins/abx_plugins/plugins/##; s#/tests##' | sort)
 
-  tests:
+          json_array="["
+          first=true
+          for plugin_name in $plugin_tests; do
+            if [ "$first" = true ]; then
+              first=false
+            else
+              json_array+=","
+            fi
+
+            needs_chromium=false
+            if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
+              needs_chromium=true
+            fi
+
+            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium}"
+          done
+          json_array+="]"
+
+          echo "plugin-tests=$json_array" >> $GITHUB_OUTPUT
+          echo "Found $(echo $plugin_tests | wc -w) plugin test suites"
+          echo "$json_array" | jq '.'
+
+  run-tests:
     name: ${{ matrix.test.name }}
+    runs-on: ubuntu-24.04
     needs: discover-tests
-    # Opt in trusted branches only; PRs always use GitHub-hosted runners.
-    # Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
-    # The hosted discovery job falls back when workers are offline or the
-    # availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
-    runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
     env:
+      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
       CHROME_HEADLESS: "true"
+      CHROME_SANDBOX: "false"
+      PERSONAS_DIR: /tmp/abx-personas
+      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
 
     strategy:
       fail-fast: false
       matrix:
-        test: ${{ fromJson(needs.discover-tests.outputs.tests) }}
+        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
+        python: ["3.13"]
 
     steps:
-      - name: Set job-local runtime paths
-        env:
-          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
-        run: |
-          workers=2
-          cache_suffix=''
-          if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
-            workers="$(nproc)"
-            cache_suffix="-workers-$workers"
-            # Avoid the hosted image's cold Chromium executable page faults.
-            # abxpkg owns installation and reuse in the existing runtime cache.
-            echo "CHROME_BINPROVIDERS=playwright,puppeteer" >> "$GITHUB_ENV"
-          fi
-          {
-            echo "CI_TEST_WORKERS=$workers"
-            echo "ABXPKG_CACHE_WORKERS_SUFFIX=$cache_suffix"
-            echo "ABXPKG_LIB_DIR=$RUNNER_TEMP/archivebox-lib"
-            echo "DATA_DIR=$RUNNER_TEMP/archivebox-test-data"
-            echo "PERSONAS_DIR=$RUNNER_TEMP/abx-personas"
-            echo "CHROME_USER_DATA_DIR=$RUNNER_TEMP/abx-personas/Default/chrome_profile"
-          } >> "$GITHUB_ENV"
-
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.CI_PYTHON_VERSION }}
+          python-version: ${{ matrix.python }}
           architecture: x64
-        id: python
 
-      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
         with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          version: "latest"
 
-      # Self-hosted tool storage survives jobs. Normal abxpkg validation and
-      # install locks allow reuse across files; cold-install tests use tmp_path.
-      - name: Reuse persistent runner tools
-        if: runner.environment == 'self-hosted'
-        env:
-          PERSISTENT_ABXPKG_LIB: ${{ runner.tool_cache }}/abxpkg/${{ github.repository }}/${{ steps.python.outputs.python-version }}
-        run: |
-          mkdir -p "$PERSISTENT_ABXPKG_LIB"
-          echo "ABXPKG_LIB_DIR=$PERSISTENT_ABXPKG_LIB" >> "$GITHUB_ENV"
-
-      # Restored binaries still pass normal abxpkg resolution/version checks.
-      # Installation and isolation tests explicitly choose fresh temporary libs.
-      - name: Identify runtime dependency cache
-        shell: bash
-        run: |
-          uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
-          import hashlib
-          import json
-          import tomllib
-          from pathlib import Path
-
-          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
-          versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
-          print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
-          PYTHON
-
-      - name: Reuse validated runtime dependencies
-        if: runner.environment != 'self-hosted'
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
         with:
-          path: ${{ runner.temp }}/archivebox-lib
-          key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}${{ env.ABXPKG_CACHE_WORKERS_SUFFIX }}
+          node-version: 22
+
+      - name: Cache uv
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
           restore-keys: |
-            abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-
+            ${{ runner.os }}-${{ matrix.python }}-uv-
 
-      - name: Resolve optional build dependencies through abxpkg
-        if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
-        env:
-          TEST_EXTRA: ${{ matrix.test.extra }}
-          TEST_PATHS: ${{ matrix.test.paths_arg }}
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.1
+
+      - name: Install dependencies with uv
         run: |
-          set -Eeuo pipefail
-          ABXPKG_SPEC="$(uv run --no-cache --no-project python -c 'import tomllib; package = next(item for item in tomllib.load(open("uv.lock", "rb"))["package"] if item["name"] == "abxpkg"); wheel = package["wheels"][0]; print("abxpkg @ {}#{}".format(wheel["url"], wheel["hash"].replace(":", "=")))')"
-          test -n "$ABXPKG_SPEC"
-          dependency_sources=()
-          if [[ -n "$TEST_EXTRA" ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
+          uv sync --dev --all-extras --no-sources --no-cache
+          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+
+      - name: Run test - ${{ matrix.test.name }}
+        run: |
+          mkdir -p tests/out
+          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
+
+  plugin-tests:
+    name: ${{ matrix.plugin.name }}
+    runs-on: ubuntu-24.04
+    needs: discover-tests
+    env:
+      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
+      CHROME_HEADLESS: "true"
+      CHROME_SANDBOX: "false"
+      PERSONAS_DIR: /tmp/abx-personas
+      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
+
+    strategy:
+      fail-fast: false
+      matrix:
+        plugin: ${{ fromJson(needs.discover-tests.outputs.plugin-tests) }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+          architecture: x64
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "latest"
+
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Cache uv
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-3.13-uv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.1
+
+      - name: Install dependencies with uv
+        run: |
+          uv sync --dev --all-extras --no-sources --no-cache
+          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+
+      - name: Select Chromium for extension-capable CDP
+        if: ${{ matrix.plugin.needs_chromium }}
+        env:
+          MIN_CHROMIUM_MAJOR: "149"
+          LIB_DIR: ${{ runner.temp }}/abx-lib
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+        run: |
+          set -euo pipefail
+          mkdir -p "$LIB_DIR"
+          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+
+          candidate="/usr/bin/chromium"
+          version=""
+          major=0
+          if [ -x "$candidate" ]; then
+            version="$("$candidate" --version || true)"
+            major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+            case "$major" in
+              ''|*[!0-9]*) major=0 ;;
+            esac
           fi
-          if [[ "$TEST_PATHS" == *test_postgres_backend* ]]; then
-            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:postgres_binaries")
+
+          if [ "$major" -ge "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "Using runner Chromium: $candidate ($version)"
+            echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+            exit 0
           fi
-          uv run --no-cache --no-project --with "$ABXPKG_SPEC" abxpkg env \
-            --install \
+
+          echo "Runner Chromium is unavailable or too old: ${version:-not found}"
+          echo "Installing Playwright Chromium into $LIB_DIR..."
+          uv run --no-sync --no-sources abxpkg install \
+            --binproviders=npm \
+            --lib="$LIB_DIR" \
+            --postinstall-scripts=false \
+            --min-release-age=0 \
             --no-cache \
-            --json \
-            --lib="$ABXPKG_LIB_DIR" \
-            "${dependency_sources[@]}"
+            --overrides='{"npm":{"install_args":["playwright@next"]}}' \
+            playwright
+          uv run --no-sync --no-sources abxpkg install \
+            --binproviders=playwright \
+            --lib="$LIB_DIR" \
+            --bin-dir="$LIB_DIR/env/bin" \
+            --min-version="$MIN_CHROMIUM_MAJOR" \
+            chromium
 
-      - name: Install test dependencies
-        env:
-          TEST_EXTRA: ${{ matrix.test.extra }}
-        run: |
-          set -Eeuo pipefail
-          extra_args=()
-          if [[ -n "$TEST_EXTRA" ]]; then
-            extra_args=(--extra "$TEST_EXTRA")
+          candidate="$LIB_DIR/env/bin/chromium"
+          if [ ! -x "$candidate" ]; then
+            echo "Playwright Chromium shim was not created at $candidate" >&2
+            exit 1
           fi
-          uv sync --no-cache --locked --dev "${extra_args[@]}"
-      - name: Run ${{ matrix.test.name }}
+          version="$("$candidate" --version || true)"
+          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
+          case "$major" in
+            ''|*[!0-9]*) major=0 ;;
+          esac
+          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
+            echo "Playwright Chromium is still too old: ${version:-unknown}" >&2
+            exit 1
+          fi
+          echo "Using Playwright Chromium: $candidate ($version)"
+          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+
+      - name: Run plugin tests - ${{ matrix.plugin.name }}
         env:
-          TEST_PATHS: ${{ matrix.test.paths_arg }}
-          CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
         run: |
-          set -Eeuo pipefail
-          echo "Running pytest with $CI_TEST_WORKERS workers on $CI_RUNNER_ENVIRONMENT"
-          read -r -a test_paths <<< "$TEST_PATHS"
-          test_args=()
-          for test_path in "${test_paths[@]}"; do
-            test -f "$test_path"
-            test_args+=("$GITHUB_WORKSPACE/$test_path")
-          done
-          # Each test still gets isolated DB/filesystem state from conftest.
-          # Keep one worker pool per batch instead of starting it for every file.
-          mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest"
-          uv run --no-cache --no-sync --no-sources pytest -vs -n "$CI_TEST_WORKERS" "${test_args[@]}" --durations=20 \
-            --basetemp="$RUNNER_TEMP/pytest" \
-            --junitxml="$RUNNER_TEMP/test-results/results.xml"
-      - name: Save test results
-        if: always()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: test-results-${{ strategy.job-index }}
-          path: ${{ runner.temp }}/test-results/*.xml
-          if-no-files-found: error
-          retention-days: 14
-      - name: Save failed runtime diagnostics
-        if: failure()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        with:
-          name: runtime-logs-${{ strategy.job-index }}
-          path: ${{ runner.temp }}/pytest/**/*.log
-          if-no-files-found: ignore
-          retention-days: 14
+          uv run --no-sync --no-sources bash ./bin/test_plugins.sh "${{ matrix.plugin.plugin }}" --no-coverage

.github/workflows/test.yml

@@ -1,60 +1,152 @@
-name: Install and CLI Compatibility
-
-on:
-  workflow_call:
-    inputs:
-      platform:
-        description: Platform for install and CLI compatibility
-        required: false
-        default: linux
-        type: string
+name: Run tests
+on: [push]
 
 env:
   DOCKER_IMAGE: archivebox-ci
-  ARCHIVEBOX_IMAGE: archivebox-ci
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
   UV_NO_SOURCES: "1"
 
 jobs:
-  install_cli_platform:
-    name: CLI install / ${{ matrix.os_name }} / Python ${{ matrix.python }}
+  python_tests:
     runs-on: ${{ matrix.os }}
+    env:
+      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
 
     strategy:
-      fail-fast: false
       matrix:
-        include: ${{ fromJSON(inputs.platform == 'macos' && '[{"os":"macos-15","os_name":"macOS","python":"3.13"},{"os":"macos-15","os_name":"macOS","python":"3.14"}]' || '[{"os":"ubuntu-24.04","os_name":"linux","python":"3.14"}]') }}
+        os: [ubuntu-22.04]
+        # os: [ubuntu-22.04, macos-latest, windows-latest]
+        python: ["3.13"]
 
     steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: actions/checkout@v4
         with:
           submodules: true
           fetch-depth: 1
 
-      ### Set up Python
+      - name: Clone abxpkg
+        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
+
+      - name: Clone abx-plugins
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
+
+      - name: Clone abx-dl
+        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
+
+      ### Setup Python & JS Languages
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
+          architecture: x64
 
       - name: Install uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: astral-sh/setup-uv@v4
         with:
-          version: "0.11.3"
-          enable-cache: false
-          cache-dependency-glob: uv.lock
+          version: "latest"
+
+      - name: Set up Node JS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      ### Install Python & JS Dependencies
+      - name: Cache uv
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/uv
+          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python }}-uv-
+
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
+          version: 1.0
 
       - name: Install dependencies with uv
-        run: uv sync --no-cache --locked --dev
-
-      - name: ArchiveBox full install check
         run: |
-          set -Eeuo pipefail
-          DATA_DIR="$PWD/tests/out/data"
-          mkdir -p "$DATA_DIR"
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox init
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox install
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox version
-          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox status
+          uv sync --dev --all-extras --no-sources --no-cache
+          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+
+      ### Run the tests
+      - name: Directory listing for debugging
+        run: |
+          pwd
+          ls
+
+      - name: Archivebox version
+        run: |
+          mkdir -p tests/out/data
+          DATA_DIR="$PWD/tests/out/data" uv run --no-sync --no-sources archivebox version
+
+      - name: Test built package with pytest
+        # TODO: remove this exception for windows once we get tests passing on that platform
+        if: ${{ !contains(matrix.os, 'windows') }}
+        run: |
+          mkdir -p tests/out
+          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs
+
+      - name: Run plugin tests
+        if: ${{ !contains(matrix.os, 'windows') }}
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
+          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
+        run: |
+          uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage
+
+  docker_tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
+      # - uses: satackey/action-docker-layer-caching@v0.0.8
+
+      - name: Build image
+        run: |
+          docker build . -t "$DOCKER_IMAGE"
+
+      - name: Init data dir
+        run: |
+          mkdir "${{ github.workspace }}/data"
+          docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init
+
+      - name: Run test server
+        run: |
+          sudo bash -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
+          docker run --name www-nginx -p 80:80 -d nginx
+
+      - name: Add link
+        run: |
+          docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local
+
+      - name: Add stdin link
+        run: |
+          echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add
+
+      - name: List links
+        run: |
+          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
+          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }
+
+      - name: Start docker-compose stack
+        run: |
+          docker-compose run archivebox init
+          docker-compose up -d
+          sleep 5
+          curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
+          curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'
+
+      - name: Check added urls show up in index
+        run: |
+          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
+          curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
+          docker-compose down || true

.gitignore

@@ -8,8 +8,6 @@ __pycache__/
 .mypy_cache/
 .eggs/
 tests/out/
-/docs/screenshots/
-/publicsite/screenshots/
 
 # Coverage
 .coverage
@@ -65,8 +63,3 @@ ArchiveBox.conf
 # vim
 *.sw?
 .vscode
-
-# Local Claude Code task locks (not project state)
-.claude/scheduled_tasks.lock
-
-/_site/

.pre-commit-config.yaml

@@ -29,12 +29,12 @@ repos:
     hooks:
       - id: ruff-format
         name: ruff-format
-        entry: uv run --active --no-sync --no-sources ruff format
+        entry: uv run --active --no-sources ruff format
         language: system
         types_or: [python, pyi]
       - id: ruff-check
         name: ruff-check
-        entry: uv run --active --no-sync --no-sources ruff check --fix
+        entry: uv run --active --no-sources ruff check --fix
         language: system
         types_or: [python, pyi]
 
@@ -56,8 +56,6 @@ repos:
       - id: mixed-line-ending
       - id: fix-byte-order-marker
       - id: end-of-file-fixer
-        # Preserve the exact bytes of captured cryptographic proof fixtures.
-        exclude: ^archivebox/tests/fixtures/legacy_proofs/.*\.json$
       - id: detect-private-key
       - id: debug-statements
       - id: forbid-submodules

AGENTS.md

@@ -1,83 +0,0 @@
-# ArchiveBox Agent Guide
-
-ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
-
-## Shared Standards
-
-- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
-- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
-- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
-- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
-- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
-- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
-- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
-- Start behavior fixes with a red failing test when a test is requested or practical.
-- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
-- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
-
-## Concurrency Contract
-
-- A collection has one orchestrator at a time. Local PID/process checks may warn about obvious same-machine duplicates, but must not claim to enforce ownership across machines or shared filesystems.
-- SQLite remains supported with concurrent short writes from CLI and server processes. Keep the SQLite database on a local filesystem, never NFS/SMB, and use short autocommit/CAS updates instead of long transactions or database locks.
-- Network calls, hook execution, filesystem migrations, and other long work belong in the orchestrator. Never hold a database transaction or lock across that work.
-- PostgreSQL and shared data directories are the path to future multi-machine scheduling. Coordinate that work through database-backed per-crawl/per-snapshot claims; do not introduce file leases or timer-based orchestrator election.
-
-## Development Setup
-
-```bash
-uv sync --dev --all-extras
-mkdir -p data
-cd data
-uv run --project .. archivebox init --install
-```
-
-Run collection commands from inside an initialized data directory:
-
-```bash
-cd data
-uv run --project .. archivebox status
-uv run --project .. archivebox add --plugins=parse_txt_urls 'https://example.com/'
-uv run --project .. archivebox run
-```
-
-## User-Facing Setup
-
-Recommended CLI install:
-
-```bash
-uv tool install --python 3.13 --prerelease explicit --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
-mkdir -p ~/archivebox/data
-cd ~/archivebox/data
-archivebox init --install
-archivebox add --plugins=parse_txt_urls 'https://example.com/'
-```
-
-Alternative install methods:
-
-- Docker Compose / Docker
-- Homebrew
-- Debian package
-- pip
-
-## Basic Usage
-
-```bash
-cd ~/archivebox/data
-archivebox version
-archivebox help
-archivebox status
-archivebox install
-archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
-archivebox list --json --with-headers
-archivebox search 'example'
-archivebox update --filter-type=domain example.com
-archivebox remove --yes --delete --filter-type=exact 'https://example.com/docs-basic-usage'
-archivebox run
-```
-
-## Verification
-
-Run the test suite with `uv run pytest archivebox/tests -x` and repository checks
-with `uv run prek run --all-files`. For focused work, select the relevant tests.
-
-Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.

CLAUDE.md

@@ -0,0 +1,497 @@
+# Claude Code Development Guide for ArchiveBox
+
+## Quick Start
+
+```bash
+# Set up dev environment (always use uv, never pip directly)
+uv sync --dev --all-extras
+
+# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
+```
+
+## Development Environment Setup
+
+### Prerequisites
+- Python 3.11+ (3.13 recommended)
+- uv package manager
+- A non-root user for running tests (e.g., `testuser`)
+
+### Install Dependencies
+```bash
+uv sync --dev --all-extras  # Always use uv, never pip directly
+```
+
+### Activate Virtual Environment
+```bash
+source .venv/bin/activate
+```
+
+### Common Gotchas
+
+#### File Permissions
+New files created by root need permissions fixed for testuser:
+```bash
+chmod 644 archivebox/tests/test_*.py
+```
+
+#### DATA_DIR Environment Variable
+ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
+
+## Code Style Guidelines
+
+### Naming Conventions for Grep-ability
+Use consistent naming for everything to enable easy grep-ability and logical grouping:
+
+**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
+
+**Examples**:
+```python
+# Filesystem migration methods - all start with fs_
+def fs_migration_needed() -> bool: ...
+def fs_migrate() -> None: ...
+def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
+def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
+def _fs_next_version(current: str) -> str: ...
+
+# Logging methods - ALL must start with log_ or _log
+def log_migration_start(snapshot_id: str) -> None: ...
+def _log_error(message: str) -> None: ...
+def log_validation_result(ok: bool, msg: str) -> None: ...
+```
+
+**Rules**:
+- Group related functions with common prefixes
+- Use `_` prefix for internal/private helpers within the same family
+- ALL logging-related methods MUST start with `log_` or `_log`
+- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
+- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
+
+### Minimize Unique Names and Data Structures
+**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
+
+**Example - GOOD**:
+```python
+# Binary has overrides field
+binary = Binary(overrides={'TIMEOUT': '60s'})
+
+# Binary reuses the same field name and structure
+class Binary(models.Model):
+    overrides = models.JSONField(default=dict)  # Same name, same structure
+```
+
+**Example - BAD**:
+```python
+# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
+class Binary(models.Model):
+    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
+```
+
+**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
+
+## Testing
+
+### CRITICAL: Never Run as Root
+ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
+
+```bash
+# Run all migration tests
+sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
+
+# Run specific test file
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
+
+# Run single test
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
+```
+
+### Test File Structure
+```
+archivebox/tests/
+├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
+├── test_migrations_fresh.py      # Fresh install tests
+├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
+├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
+└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
+```
+
+### Test Writing Standards
+
+#### NO MOCKS - Real Tests Only
+Tests must exercise real code paths:
+- Create real SQLite databases with version-specific schemas
+- Seed with realistic test data
+- Run actual `python -m archivebox` commands via subprocess
+- Query SQLite directly to verify results
+
+**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
+
+#### NO SKIPS
+Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
+
+#### Strict Assertions
+- `init` command must return exit code 0 (not `[0, 1]`)
+- Verify ALL data is preserved, not just "at least one"
+- Use exact counts (`==`) not loose bounds (`>=`)
+
+### Example Test Pattern
+```python
+def test_migration_preserves_snapshots(self):
+    """Migration should preserve all snapshots."""
+    result = run_archivebox(self.work_dir, ['init'], timeout=45)
+    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
+
+    ok, msg = verify_snapshot_count(self.db_path, expected_count)
+    self.assertTrue(ok, msg)
+```
+
+### Testing Gotchas
+
+#### Extractors Disabled for Speed
+Tests disable all extractors via environment variables for faster execution:
+```python
+env['SAVE_TITLE'] = 'False'
+env['SAVE_FAVICON'] = 'False'
+# ... etc
+```
+
+#### Timeout Settings
+Use appropriate timeouts for migration tests (45s for init, 60s default).
+
+### Plugin Testing & Code Coverage
+
+**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
+
+```bash
+# Run plugin tests with coverage (both Python + JavaScript)
+bash bin/test_plugins.sh screenshot
+
+# View coverage reports
+bash bin/test_plugins.sh --coverage-report
+# Or individual reports:
+coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
+```
+
+#### Plugin Test Structure
+
+Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
+
+```python
+# Correct production paths:
+# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
+# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
+
+with tempfile.TemporaryDirectory() as tmpdir:
+    data_dir = Path(tmpdir)
+
+    # Crawl-level plugin (e.g., chrome launcher)
+    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
+    chrome_dir = crawl_dir / 'chrome'
+    chrome_dir.mkdir(parents=True)
+
+    # Snapshot-level plugin (e.g., screenshot)
+    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
+    screenshot_dir = snapshot_dir / 'screenshot'
+    screenshot_dir.mkdir(parents=True)
+
+    # Run hook in its output directory
+    result = subprocess.run(
+        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
+        cwd=str(screenshot_dir),
+        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
+        capture_output=True,
+        timeout=120
+    )
+```
+
+#### Coverage Improvement Loop
+
+To improve from ~20% to 80%+:
+
+1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
+2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
+3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
+4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
+
+**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
+
+## Database Migrations
+
+### Generate and Apply Migrations
+```bash
+# Generate migrations (run from archivebox subdirectory)
+cd archivebox
+./manage.py makemigrations
+
+# Apply migrations to test database
+cd data/
+archivebox init
+```
+
+### Schema Versions
+- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
+- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
+- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
+- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
+
+### Testing a Migration Path
+1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
+2. Seed with realistic test data using `seed_0_X_data()`
+3. Run `archivebox init` to trigger migrations
+4. Verify data preservation with `verify_*` functions
+5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
+
+### Squashed Migrations
+When testing 0.8.x (dev branch), you must record ALL replaced migrations:
+```python
+# The squashed migration replaces these - all must be recorded
+('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
+('core', '0024_auto_20240513_1143'),
+# ... all 52 migrations from 0023-0074 ...
+('core', '0023_new_schema'),  # Also record the squashed migration itself
+```
+
+### Migration Strategy
+- Squashed migrations for clean installs
+- Individual migrations recorded for upgrades from dev branch
+- `replaces` attribute in squashed migrations lists what they replace
+
+### Migration Gotchas
+
+#### Circular FK References in Schemas
+SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
+
+## Plugin System Architecture
+
+### Plugin Dependency Rules
+
+Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
+However, they are allowed to depend on two shared files ONLY:
+- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
+- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
+
+### Chrome-Dependent Plugins
+
+Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
+
+**Main puppeteer-based chrome installer + launcher plugin**:
+- `chrome` - Core Chrome integration (CDP, launch, navigation)
+
+**Metadata extraction using chrome/chrome_utils.js / CDP**:
+- `dns` - DNS resolution info
+- `ssl` - SSL certificate info
+- `headers` - HTTP response headers
+- `redirects` - Capture redirect chains
+- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
+- `responses` - Capture network responses
+- `consolelog` - Capture console.log output
+- `title` - Extract page title
+- `accessibility` - Extract accessibility tree
+- `seo` - Extract SEO metadata
+
+**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
+- `ublock` - uBlock Origin ad blocking
+- `istilldontcareaboutcookies` - Cookie banner dismissal
+- `twocaptcha` - 2captcha CAPTCHA solver integration
+
+**Page-alteration plugins to prepare the content for archiving**:
+- `modalcloser` - Modal dialog dismissal
+- `infiniscroll` - Infinite scroll handler
+
+**Main Extractor Outputs**:
+- `dom` - DOM snapshot extraction
+- `pdf` - Generate PDF snapshots
+- `screenshot` - Generate screenshots
+- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
+
+**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
+- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
+- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
+- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
+- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
+
+### Finding Chrome-Dependent Plugins
+
+```bash
+# Find all files containing "chrom" (case-insensitive)
+grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
+
+# Or get just the plugin names
+grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
+```
+
+**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
+
+## Architecture Notes
+
+### Crawl Model (0.9.x)
+- Crawl groups multiple Snapshots from a single `add` command
+- Each `add` creates one Crawl with one or more Snapshots
+- Seed model was removed - crawls now store URLs directly
+
+## Code Coverage
+
+### Overview
+
+Coverage tracking is enabled for passive collection across all contexts:
+- Unit tests (pytest)
+- Integration tests
+- Dev server (manual testing)
+- CLI usage
+
+Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
+
+### Install Coverage Tools
+
+```bash
+uv sync --dev  # Installs pytest-cov and coverage
+```
+
+### Running with Coverage
+
+#### Unit Tests
+```bash
+# Run tests with coverage
+pytest --cov=archivebox --cov-report=term archivebox/tests/
+
+# Or run specific test file
+pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
+```
+
+#### Dev Server with Coverage
+```bash
+# Start dev server with coverage tracking
+coverage run --parallel-mode -m archivebox server
+
+# Or CLI commands
+coverage run --parallel-mode -m archivebox init
+coverage run --parallel-mode -m archivebox add https://example.com
+```
+
+#### Manual Testing (Always-On)
+To enable coverage during ALL Python executions (passive tracking):
+
+```bash
+# Option 1: Use coverage run wrapper
+coverage run --parallel-mode -m archivebox [command]
+
+# Option 2: Set environment variable (tracks everything)
+export COVERAGE_PROCESS_START=pyproject.toml
+# Now all Python processes will track coverage
+archivebox server
+archivebox add https://example.com
+```
+
+### Viewing Coverage
+
+#### Text Report (Quick View)
+```bash
+# Combine all parallel coverage data
+coverage combine
+
+# View summary
+coverage report
+
+# View detailed report with missing lines
+coverage report --show-missing
+
+# View specific file
+coverage report --include="archivebox/core/models.py" --show-missing
+```
+
+#### JSON Report (LLM-Friendly)
+```bash
+# Generate JSON report
+coverage json
+
+# View the JSON
+cat coverage.json | jq '.files | keys'  # List all files
+
+# Find files with low coverage
+cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
+
+# Find completely uncovered files (dead code candidates)
+cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
+
+# Get missing lines for a specific file
+cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
+```
+
+#### HTML Report (Visual)
+```bash
+# Generate interactive HTML report
+coverage html
+
+# Open in browser
+open htmlcov/index.html
+```
+
+### Isolated Runs
+
+To measure coverage for specific scenarios:
+
+```bash
+# 1. Reset coverage data
+coverage erase
+
+# 2. Run your isolated test/scenario
+pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
+# OR
+coverage run --parallel-mode -m archivebox add https://example.com
+
+# 3. View results
+coverage combine
+coverage report --show-missing
+
+# 4. Optionally export for analysis
+coverage json
+```
+
+### Finding Dead Code
+
+```bash
+# 1. Run comprehensive tests + manual testing to build coverage
+pytest --cov=archivebox archivebox/tests/
+coverage run --parallel-mode -m archivebox server  # Use the app manually
+coverage combine
+
+# 2. Find files with 0% coverage (strong dead code candidates)
+coverage json
+cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
+
+# 3. Find files with <10% coverage (likely dead code)
+cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
+
+# 4. Generate detailed report for analysis
+coverage report --show-missing > coverage_report.txt
+```
+
+### Tips
+
+- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
+- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
+- **Reset**: Use `coverage erase` to start fresh for isolated measurements
+- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
+- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
+
+## Debugging Tips
+
+### Check Migration State
+```bash
+sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
+```
+
+### Check Table Schema
+```bash
+sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
+```
+
+### Verbose Test Output
+```bash
+sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
+```
+
+### Kill Zombie Chrome Processes
+```bash
+./bin/kill_chrome.sh
+```

Dockerfile

@@ -1,230 +1,38 @@
-# syntax=docker/dockerfile:1.7
+# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
+#     python3.13, uv, python3-ldap
+#     curl, wget, git, dig, ping, tree, nano
+#     node, npm
+#     ArchiveBox and plugin runtime dependencies installed by archivebox init --install
+# Usage:
+#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
+#     docker build . -t archivebox
+#     docker run -v "$PWD/data":/data archivebox init
+#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
+#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
+#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
+#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
+# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
 
-# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# WHY: abx-dl owns Python, Node, Chromium, and extraction dependencies so the
-# standalone downloader stays independent of ArchiveBox's server features.
-# ArchiveBox-only dependencies (server plugins, supervisor, Django) belong in
-# the layers added by this Dockerfile, never in the shared abx-dl base image.
-# REQUIRED INSTALLATION CONTRACT — do not work around failures:
-# 1. IMAGE CONSTRUCTION: preinstall ALL plugins supported by runtime="archivebox"
-#    and ALL their required_binaries, regardless of default enabled state.
-#    Inherit the complete downloader dependencies from the tested abx-dl image;
-#    install every remaining ArchiveBox-only dependency in this image's layers,
-#    including optional server plugins. Never put server dependencies in abx-dl.
-#    Use the normal plugin dependency pipeline: install command -> config.json
-#    required_binaries -> abxpkg. Never add manual apt/pip/npm installs to hide
-#    failures in that pipeline. OS/interpreter bootstrap is a separate concern,
-#    not a second hand-maintained list of plugin dependencies.
-# 2. FINISHED IMAGE: archivebox install and the full plugin installation check
-#    must discover everything already present and be NO-OPS offline as UID 911.
-#    No downloads, package installs, sudo, or root repair may be needed to enable
-#    ANY plugin or start a new crawl. Never run verification as root or manually
-#    preinstall packages before it. If the inherited image lacks a downloader
-#    dependency, fix/release abx-dl and consume that tested image; do not patch
-#    over the missing base dependency here or repair deployed containers by SSH.
-# 3. NON-CONTAINER CI: exercise real runtime auto-installation through the same
-#    CLI/required_binaries/abxpkg path users invoke. GitHub's runner has
-#    passwordless sudo; abxpkg must escalate automatically when needed. Never
-#    manually preinstall dependencies or wrap the tested command in sudo to make
-#    a failure disappear. This is distinct from the finished-image no-op test.
-# 4. SIZE: the complete compressed image budget is 1025 MiB. Preserve ALL
-#    user-facing functionality. Only mechanical/internal
-#    optimizations are allowed. Never remove dependencies, disable features or
-#    plugins, weaken assertions, or skip checks to meet a size limit. If the
-#    complete image exceeds the configured limit, LET CI FAIL and explicitly
-#    ask the maintainer what to do. Do not raise the limit unilaterally.
-# Build abx-dl first, then point this file at it:
-#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
-#       --build-context abxbus=../abxbus \
-#       --build-context abxpkg=../abxpkg \
-#       --build-context abx-plugins=../abx-plugins \
-#       -t archivebox/abx-dl:dev
-#   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:dev \
-#       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:main
+#########################################################################################
 
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
+### Example: Using ArchiveBox in your own project's Dockerfile ########
+
+# FROM python:3.13-slim
+# WORKDIR /data
+# RUN pip install archivebox>=0.9.0   # use latest release here
+# RUN archivebox install
+# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
+
+#########################################################################################
 
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-ARG TARGETVARIANT
-ARG ARCHIVEBOX_COMMIT_HASH=""
+ARG TARGETVARIANT=
 
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_NO_COMPILE=1 \
-    PIP_ONLY_BINARY=aiohttp \
-    npm_config_loglevel=error
-
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=24
-
-ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_ARCHIVEBOX_UID=911 \
-    DEFAULT_ARCHIVEBOX_GID=911 \
-    IN_DOCKER=True
-
-ENV CODE_DIR=/app \
-    ABX_RUNTIME=archivebox \
-    DATA_DIR=/data \
-    CONFIG_DIR=/opt/archivebox \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
-    PERSONAS_DIR=/data/personas \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/venv/bin/python3 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-ENV HOME=/home/archivebox \
-    XDG_CONFIG_HOME=/home/archivebox/.config \
-    XDG_CACHE_HOME=/opt/archivebox/lib/cache \
-    TIMEOUT=600
-
-ENV UV_COMPILE_BYTECODE=false \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv \
-    VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:$PATH"
-
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
-WORKDIR "$CODE_DIR"
-
-RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
-    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env node \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env uv \
-    ) | tee -a /VERSION.txt
-
-FROM archivebox-runtime-base AS archivebox-builder
-
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=bind,source=uv.lock,target=/app/uv.lock \
-    <<'EOF'
-echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
-export UV_NO_CACHE=true
-rm -f /etc/apt/apt.conf.d/docker-clean
-apt-get update -qq
-apt-get install -qq -y --no-install-recommends x11-utils
-# Extend abx-dl's existing venv instead of clearing it. Clearing and later
-# copying a complete replacement over the base duplicates every venv byte in
-# the final overlay history even when most packages are unchanged.
-# Do not rewrite inherited native libraries merely to strip them again.
-/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print > /tmp/archivebox-inherited-libraries
-
-# The old Docker-only dependency rewrite predated this checkout's lockfile:
-# it fetched version JSON to avoid stale PyPI Simple listings. That extra API
-# can itself lag or return 503s. The verified lock already records exact public
-# artifact URLs and hashes, including extras and platform markers. Consume it
-# directly, retaining the freshness check instead of resolving a second graph.
-/usr/bin/uv sync \
-    --project /app \
-    --locked \
-    --no-cache \
-    --no-dev \
-    --inexact \
-    --no-install-project \
-    --no-install-workspace \
-    --no-install-package abxbus \
-    --no-install-package abxpkg \
-    --no-install-package abx-plugins \
-    --no-install-package abx-dl \
-    --no-sources
-# Sync may replace inherited dependencies (e.g. pydantic/rich-click), removing
-# their bytecode. Runtime writes are disabled, so missing caches would make
-# every hook compile them again. Compile only missing entries: compileall's
-# timestamp fast path rewrites even valid checked-hash caches, copying the
-# entire inherited venv into this OCI layer for no runtime benefit.
-/usr/bin/uv run --no-project --no-sync python - <<'PY'
-import importlib.util
-from pathlib import Path
-import py_compile
-import sysconfig
-
-for source in Path(sysconfig.get_path("purelib")).rglob("*.py"):
-    cache = Path(importlib.util.cache_from_source(str(source)))
-    if not cache.is_file():
-        py_compile.compile(str(source), doraise=True, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH)
-PY
-builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
-ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
-/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
-while IFS= read -r -d '' native_library; do
-    if grep -Fxq "$native_library" /tmp/archivebox-inherited-libraries; then
-        continue
-    fi
-    magic=''
-    if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
-        "$builder_abxpkg_lib_dir/env/bin/strip" --strip-unneeded "$native_library" || exit $?
-    fi
-done < /tmp/archivebox-native-libraries
-rm -f /tmp/archivebox-native-libraries /tmp/archivebox-inherited-libraries
-rm -f /venv/bin/uv /venv/bin/uvx
-abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get purge -y binutils
-abxpkg run --binproviders=env --lib="$builder_abxpkg_lib_dir" apt-get autoremove -y
-rm -rf "$builder_abxpkg_lib_dir"
-rm -rf /venv/lib/python3.*/site-packages/pip* \
-    /venv/lib/python3.*/site-packages/wheel* \
-    /venv/bin/pip /venv/bin/pip3 /venv/bin/pip3.* /venv/bin/wheel
-rm -rf /var/lib/apt/lists/*
-EOF
-
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-# Compile only ArchiveBox's installed package. Touching all of /venv here would
-# copy every inherited abx-dl file into a new layer merely to change its mtime.
-RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && COMMIT_HASH="$( \
-        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-            echo "$ARCHIVEBOX_COMMIT_HASH"; \
-        elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
-            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
-            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-                echo "$HEAD_REF"; \
-            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
-                REF_PATH="${HEAD_REF#ref: }"; \
-                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
-            fi; \
-        fi)" \
-    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
-    && /usr/bin/uv pip install --no-cache --no-deps "$CODE_DIR" \
-    && rm -f /venv/bin/uv /venv/bin/uvx \
-    && cd / \
-    && ARCHIVEBOX_PY_DIR="$(/venv/bin/python -c 'import pathlib, archivebox; print(pathlib.Path(archivebox.__file__).parent)')" \
-    && /venv/bin/python -m compileall --invalidation-mode checked-hash -q "$ARCHIVEBOX_PY_DIR" \
-    && test -f "$(/venv/bin/python -c 'import archivebox; print(archivebox.__cached__)')" \
-    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
-
-# The builder installs and purges stripping tools in one layer, so its final
-# filesystem is already runtime-clean. Preserve that ancestry: starting again
-# from archivebox-runtime-base and COPYing /venv would bake a second full venv
-# over the inherited one instead of recording only ArchiveBox's package delta.
-FROM archivebox-builder
+FROM archivebox/sonic:1.4.9 AS sonic
+FROM ubuntu:24.04
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -235,93 +43,353 @@ LABEL name="archivebox" \
     org.opencontainers.image.vendor="ArchiveBox" \
     org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
     org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile"
+    com.docker.image.source.entrypoint="Dockerfile" \
+    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
+    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
+    com.docker.desktop.extension.api.version=">= 1.4.7" \
+    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
+    com.docker.extension.publisher-url="https://archivebox.io" \
+    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
+    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
+    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
+    com.docker.extension.categories='database,utility-tools'
 
+ARG TARGETPLATFORM
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
+######### Environment Variables #################################
+
+# Global build-time and runtime environment constants + default pkg manager config
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    npm_config_loglevel=error
+
+# Language Version config
+ENV PYTHON_VERSION=3.13 \
+    NODE_VERSION=22.22.3
+
+# Non-root User config
+ENV ARCHIVEBOX_USER="archivebox" \
+    DEFAULT_PUID=911 \
+    DEFAULT_PGID=911 \
+    IN_DOCKER=True
+
+# ArchiveBox Source Code + Lib + Data paths
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    LIB_DIR=/opt/archivebox/lib \
+    LIB_BIN_DIR=/opt/archivebox/lib/bin \
+    ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    PLAYWRIGHT_BROWSERS_PATH=/browsers
+
+# Bash SHELL config
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
+
+######### System Environment ####################################
+
+# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
+WORKDIR "$CODE_DIR"
+
+# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
+RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
+    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
+    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
+    && rm -f /etc/apt/apt.conf.d/docker-clean
+
+# Print debug info about build and save it to disk, for human eyes only, not used by anything else
+RUN (echo "[i] Docker build for ArchiveBox starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && echo \
+    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
+    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
+    && echo \
+    && uname -a \
+    && sed -n '1,7p' /etc/os-release \
+    && which bash && bash --version | sed -n '1p' \
+    && which dpkg && dpkg --version | sed -n '1p' \
+    && echo -e '\n\n' && env && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+# Create non-privileged user for archivebox and chrome
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
+    && groupadd --system $ARCHIVEBOX_USER \
+    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
+    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
+    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
+    | tee -a /VERSION.txt
+    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
+    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
+
+# Install system apt dependencies (adding backports to access more recent apt updates)
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
+    && mkdir -p /etc/apt/keyrings \
+    && apt-get update -qq \
+    && apt-get install -qq -y \
+        # 1. packaging dependencies
+        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
+        # 2. docker and init system dependencies
+        zlib1g-dev dumb-init gosu cron unzip grep dnsutils git python3.12-venv \
+        # 3. frivolous CLI helpers to make debugging failed archiving easier
+        tree nano iputils-ping \
+        # nano iputils-ping dnsutils htop procps jq yq
+    && rm -rf /var/lib/apt/lists/*
+
+# Install sonic search backend
 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
+RUN (which sonic && sonic --version) | tee -a /VERSION.txt
 
-# The builder invokes abxpkg as root for temporary ELF stripping, which can
-# rewrite derived state ownership. Restore UID 911 before resolving binaries
-# as the runtime user; the old copy-based final stage hid this dependency.
-RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777,id=archivebox-runtime-config-$TARGETARCH \
-    echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
-    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
-    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
-    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
-    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
-    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
-    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
-    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
-    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
-    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
-    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
-    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
-    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
-    && find "$ABXPKG_LIB_DIR" \( ! -user "$DEFAULT_ARCHIVEBOX_UID" -o ! -group "$DEFAULT_ARCHIVEBOX_GID" \) -exec chown -h "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" {} + \
-    && UV_NO_CACHE=true setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
-    && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+######### Language Environments ####################################
 
-# Install the complete ArchiveBox catalog, including disabled server plugins.
-# Explicit plugin names make installation independent of enabled defaults.
-RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-server-plugins-$TARGETARCH \
-    chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" /var/tmp/abxpkg-cache \
-    && chmod 1777 /var/tmp/abxpkg-cache \
-    && export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
-    && env ABXPKG_TMP_CACHE_DIR=/var/tmp/abxpkg-cache XDG_CACHE_HOME=/var/tmp/abxpkg-cache \
-        setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
-    && UV_NO_CACHE=true /usr/bin/uv run --no-project python -c 'from abx_plugins.plugins.opencode.image import prune_incompatible_image_files; prune_incompatible_image_files()' \
-    # pnpm's explicit store lives under ABXPKG_LIB_DIR, regardless of XDG_CACHE_HOME.
-    # Retaining its download copies made image size depend on whether the build
-    # preserved hardlinks to installed files. Remove the store in this layer;
-    # installed files retain their contents, and the offline check below proves
-    # the complete catalog still works without the disposable package payloads.
-    && rm -rf "$ABXPKG_LIB_DIR/cache/pnpm"
-RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
-    bash -c 'uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_installed; verify_installed()" && abx-dl install $ABX_DOCKER_PLUGINS'
+# Set up Python environment
+# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
+#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
+    # && apt-get update -qq \
+    # && apt-get install -qq -y --no-upgrade \
+    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
+    # && rm -rf /var/lib/apt/lists/* \
+    # tell PDM to allow using global system python site packages
+    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
+    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
+    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
+    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
+    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
+    # && python3 -m venv $GLOBAL_VENV \
+    # install global dependencies / python build dependencies in GLOBAL_VENV
+    # && pip install --upgrade pip setuptools wheel \
+    # Save version info
+    # && ( \
+    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
+    #     && which pip && pip --version \
+    #     # && which pdm && pdm --version \
+    #     && echo -e '\n\n' \
+    # ) | tee -a /VERSION.txt
+
+
+# Set up Node environment from the official platform tarball. This avoids
+# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
+# emulated Docker builds.
+RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    case "$TARGETARCH" in \
+        amd64) NODE_DIST_ARCH="x64" ;; \
+        arm64) NODE_DIST_ARCH="arm64" ;; \
+        *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \
+    esac \
+    && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \
+    && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \
+    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \
+    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \
+    && grep "  ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \
+    && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \
+    && rm "$NODE_TARBALL" SHASUMS256.txt \
+    # Save version info
+    && ( \
+        which node && node --version \
+        && which npm && npm --version \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+
+# Set up uv and main app /venv
+RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
+ENV UV_COMPILE_BYTECODE=0 \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv
+WORKDIR "$CODE_DIR"
+# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
+    && uv venv /venv --python ${PYTHON_VERSION}
+ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
+RUN uv pip install setuptools pip \
+    && ( \
+        which python3 && python3 --version \
+        && which uv && uv self version \
+        && uv python find --system && uv python find \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt
+
+
+######### ArchiveBox & Extractor Dependencies ##################################
+
+# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
+    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        build-essential gcc \
+        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
+        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
+    && uv pip install \
+        "python-ldap>=3.4.3" \
+    && apt-get purge -y \
+        python3-dev build-essential gcc \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
+
+
+# Runtime config used by plugin hooks. Plugin binaries and npm packages are
+# installed into LIB_DIR below by archivebox init --install and resolved from
+# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
+ENV PERSONAS_DIR=/data/personas \
+    CHROME_EXTENSIONS_DIR=/opt/archivebox/lib/chrome_extensions \
+    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
+    CHROME_HEADLESS=true \
+    CHROME_SANDBOX=false \
+    CHROME_ISOLATION=crawl \
+    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+
+######### Build Dependencies ####################################
+
+
+# Install ArchiveBox Python venv dependencies from pyproject.toml.
+RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \
+    && uv --no-cache sync \
+        --refresh \
+        --no-dev \
+        --inexact \
+        --all-extras \
+        --no-install-project \
+        --no-install-workspace \
+        --no-sources \
+    && apt-get purge -y python3-dev build-essential gcc \
+    && apt-get autoremove -y \
+    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && rm -rf /var/lib/apt/lists/*
+    # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies
+
+# Setup ArchiveBox runtime config
+ENV TMP_DIR=/tmp/archivebox \
+    PIP_VENV_PYTHON=/usr/bin/python3.12 \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
 
+WORKDIR "$DATA_DIR"
+RUN openssl rand -hex 16 > /etc/machine-id \
+    && mkdir -p "$DATA_DIR" \
+    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \
+    && mkdir -p "$TMP_DIR" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
+    && mkdir -p "$LIB_DIR" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
+    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
+    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+
+# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer
+# path users run later, before copying ArchiveBox source so source-only edits do
+# not invalidate the heavy browser/plugin dependency layer.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
+    --mount=type=cache,target=/opt/archivebox/lib,sharing=locked,id=archivebox-lib-$TARGETARCH$TARGETVARIANT \
+    echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
+    && export PERSONAS_DIR="$LIB_DIR/personas" \
+    && export CHROME_EXTENSIONS_DIR="$LIB_DIR/chrome_extensions" \
+    && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
+    && mkdir -p "$LIB_DIR" "$LIB_DIR/chrome_extensions" \
+    && apt-get update -qq \
+    && if [ "$TARGETARCH" = "arm64" ]; then \
+        abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
+        abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
+    fi \
+    && TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \
+    && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \
+    && mkdir -p /opt/archivebox/lib-layer \
+    && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \
+    && rm -rf /var/lib/apt/lists/* \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer
+
+RUN rm -rf "$LIB_DIR" \
+    && mv /opt/archivebox/lib-layer "$LIB_DIR" \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
+
+# Install ArchiveBox Python package from the checked-out source.
+WORKDIR "$CODE_DIR"
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
+    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && pip install \
+        --no-deps \
+        "$CODE_DIR" \
+    && ( \
+        pip show archivebox \
+        && which archivebox \
+        && echo -e '\n\n' \
+    ) | tee -a /VERSION.txt \
+    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
+    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
+
+# Initialize an empty image collection without rerunning dependency installs.
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
-    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && HOME="$TMP_DIR" archivebox init \
-    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
+    && PUID=0 PGID=0 archivebox init \
+    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
+    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
         "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
         "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
         "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
         "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
-        "$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
-        2>/dev/null || true) \
-    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+        2>/dev/null || true)
 
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
-
-RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
-    && test -f "$(/venv/bin/python -c 'import pydantic; print(pydantic.__cached__)')" \
-    && test -f "$(/venv/bin/python -c 'import rich_click; print(rich_click.__cached__)')" \
-    && test -f "$(/venv/bin/python -c 'import platformdirs; print(platformdirs.__cached__)')" \
-    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+# Print version for nice docker finish summary
+RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
     && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
     ) | tee -a /VERSION.txt
 
+# Verify ArchiveBox is installed and write full version/dependency info.
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
+    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
+    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
+    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
+    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
+    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
+    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+
+####################################################
+
+# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
-EXPOSE 5797
+EXPOSE 8000
 
 HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:5797/health/' | grep -q 'OK'
+    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
 
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:5797"]
+CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2026 Nick Sweeting
+Copyright (c) 2024 Nick Sweeting
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

archivebox/__init__.py

@@ -38,6 +38,11 @@ ASCII_LOGO = """
 
 PACKAGE_DIR = Path(__file__).resolve().parent
 
+# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models
+# # Migrations reference models like 'machine.Binary' which need to be importable
+# if str(PACKAGE_DIR) not in sys.path:
+#     sys.path.append(str(PACKAGE_DIR))
+
 os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
 os.environ["TZ"] = "UTC"
 
@@ -46,9 +51,10 @@ from .config.permissions import drop_privileges  # noqa
 
 drop_privileges()
 
-from .misc.checks import check_not_root, check_io_encoding  # noqa
+from .misc.checks import check_not_root, check_not_inside_source_dir, check_io_encoding  # noqa
 
 check_not_root()
+check_not_inside_source_dir()
 check_io_encoding()
 
 from .config.version import VERSION  # noqa
@@ -73,10 +79,16 @@ def __getattr__(name: str):
         return VERSION
     if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
         from abx_plugins import get_plugins_dir
-        from .config.constants import CONSTANTS
 
         builtin_plugins_dir = Path(get_plugins_dir()).resolve()
-        user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
+        user_plugins_dir = (
+            Path(
+                os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR")
+                or os.environ.get("USER_PLUGINS_DIR")
+                or os.environ.get("DATA_DIR", os.getcwd()),
+            )
+            / "custom_plugins"
+        )
         plugins = {
             "builtin": builtin_plugins_dir,
             "user": user_plugins_dir,

archivebox/__main__.py

@@ -17,4 +17,4 @@ ASCII_LOGO_MINI = r"""
 """
 
 if __name__ == "__main__":
-    main(args=sys.argv[1:])
+    main(args=sys.argv[1:], stdin=sys.stdin)

archivebox/api/apps.py

@@ -7,11 +7,6 @@ class APIConfig(AppConfig):
     name = "archivebox.api"
     label = "api"
 
-    def ready(self):
-        from archivebox.api.webhooks import register_delete_webhook_field_loader
-
-        register_delete_webhook_field_loader()
-
 
 def register_admin(admin_site):
     from archivebox.api.admin import register_admin

archivebox/api/auth.py

@@ -51,27 +51,6 @@ def auth_using_token(token: str | None, request: HttpRequest | None = None) -> U
     return user
 
 
-def token_from_request(request: HttpRequest) -> str:
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-    return token
-
-
-def authenticated_user_from_request(request: HttpRequest) -> User | None:
-    user = request.user
-    if user.is_authenticated and user.is_active:
-        return user
-
-    token = token_from_request(request)
-    token_user = auth_using_token(token=token, request=request) if token else None
-    if token_user and token_user.is_active:
-        request.user = token_user
-        return token_user
-    return None
-
-
 def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
     """Given a username and password, check if they are valid and return the corresponding user"""
     user: User | None = None

archivebox/api/migrations/0001_initial.py

@@ -12,9 +12,19 @@ import signal_webhooks.fields
 import signal_webhooks.utils
 
 
-# Raw sqlite DDL kept EXACTLY as-is; executed only on sqlite via Django's
-# RunSQL (identical statement splitting → byte-for-byte identical behavior).
-INITIAL_SQL = """
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        ("auth", "0012_alter_user_first_name_max_length"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunSQL(
+                    sql="""
                 -- Create api_apitoken table
                 CREATE TABLE IF NOT EXISTS api_apitoken (
                     id TEXT PRIMARY KEY NOT NULL,
@@ -60,60 +70,12 @@ INITIAL_SQL = """
                 CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
                 CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
                 CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
-                    """
-
-INITIAL_REVERSE_SQL = """
+                    """,
+                    reverse_sql="""
                 DROP TABLE IF EXISTS api_outboundwebhook;
                 DROP TABLE IF EXISTS api_apitoken;
-                    """
-
-
-def _run_sqlite_only_sql(apps, schema_editor):
-    if schema_editor.connection.vendor != "sqlite":
-        return
-    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_forwards(
-        "api",
-        schema_editor,
-        None,
-        None,
-    )
-
-
-def _run_sqlite_only_sql_reverse(apps, schema_editor):
-    if schema_editor.connection.vendor != "sqlite":
-        return
-    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_backwards(
-        "api",
-        schema_editor,
-        None,
-        None,
-    )
-
-
-def _pg_sync_schema(apps, schema_editor):
-    from archivebox.misc.db import rebuild_models_from_migration_state
-
-    rebuild_models_from_migration_state(apps, schema_editor, "api", ["APIToken", "OutboundWebhook"])
-
-
-def _pg_drop_schema(apps, schema_editor):
-    from archivebox.misc.db import drop_models_on_postgres
-
-    drop_models_on_postgres(apps, schema_editor, "api", ["OutboundWebhook", "APIToken"])
-
-
-class Migration(migrations.Migration):
-    initial = True
-
-    dependencies = [
-        ("auth", "0012_alter_user_first_name_max_length"),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.SeparateDatabaseAndState(
-            database_operations=[
-                migrations.RunPython(_run_sqlite_only_sql, reverse_code=_run_sqlite_only_sql_reverse),
+                    """,
+                ),
             ],
             state_operations=[
                 migrations.CreateModel(
@@ -274,8 +236,4 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
-        # On non-sqlite backends the raw DDL above is skipped, so the real
-        # schema diverges from Django state. Rebuild both tables (including the
-        # unique constraint) from the post-migration state (no-op on sqlite).
-        migrations.RunPython(_pg_sync_schema, reverse_code=_pg_drop_schema),
     ]

archivebox/api/models.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox.api"
 
 import secrets
-from archivebox.uuid_compat import CompactUUIDField, uuid7
+from archivebox.uuid_compat import uuid7
 
 from django.conf import settings
 from django.db import models
@@ -17,7 +17,7 @@ def generate_secret_token() -> str:
 
 
 class APIToken(models.Model):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
@@ -41,7 +41,7 @@ class APIToken(models.Model):
 
 
 class OutboundWebhook(WebhookBase):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)

archivebox/api/urls.py

@@ -7,7 +7,7 @@ from django.shortcuts import redirect
 from django.urls import path
 from django.views.generic.base import RedirectView
 
-from archivebox.core.routes_util import build_web_url
+from archivebox.core.host_utils import build_web_url
 from .v1_api import urls as v1_api_urls
 
 
@@ -23,4 +23,7 @@ urlpatterns = [
     path("v1/", RedirectView.as_view(url="/api/v1/docs")),
     path("v1/", v1_api_urls),
     path("v1", RedirectView.as_view(url="/api/v1/docs")),
+    # ... v2 can be added here ...
+    # path("v2/",              v2_api_urls),
+    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
 ]

archivebox/api/v1_api.py

@@ -2,8 +2,6 @@ __package__ = "archivebox.api"
 
 
 from io import StringIO
-from functools import lru_cache
-from typing import Any
 from traceback import format_exception
 from contextlib import redirect_stdout, redirect_stderr
 
@@ -12,7 +10,6 @@ from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, Permissio
 from django.contrib.auth.models import User
 
 from ninja import NinjaAPI, Swagger
-from ninja.openapi.schema import OpenAPISchema
 
 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
 
@@ -52,16 +49,6 @@ def register_urls(api: NinjaAPI) -> NinjaAPI:
 
 
 class NinjaAPIWithIOCapture(NinjaAPI):
-    def get_openapi_schema(self, *, path_prefix: str | None = None, path_params: dict[str, Any] | None = None) -> OpenAPISchema:
-        prefix = path_prefix if path_prefix is not None else self.get_root_path(path_params or {})
-        return self._schema_for_prefix(prefix)
-
-    @lru_cache(maxsize=8)
-    def _schema_for_prefix(self, path_prefix: str) -> OpenAPISchema:
-        # Routes are registered at startup. Server discovery can reuse their
-        # schema while preserving distinct URL prefixes for mounted instances.
-        return super().get_openapi_schema(path_prefix=path_prefix)
-
     def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
         stdout, stderr = StringIO(), StringIO()
 
@@ -81,16 +68,19 @@ class NinjaAPIWithIOCapture(NinjaAPI):
         # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
 
         # Add Auth Headers to response
-        api_token_attr = request.__dict__.get("_api_token")
+        api_token_attr = getattr(request, "_api_token", None)
         api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
         token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
 
-        response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
+        response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
         response["X-ArchiveBox-Auth-Expires"] = token_expiry
         response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
+        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
         response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
 
+        # import ipdb; ipdb.set_trace()
+        # print('RESPONDING NOW', response)
+
         return response
 
 
@@ -101,6 +91,8 @@ api = NinjaAPIWithIOCapture(
     auth=API_AUTH_METHODS,
     urls_namespace="api-1",
     docs=Swagger(settings={"persistAuthorization": True}),
+    # docs_decorator=login_required,
+    # renderer=ORJSONRenderer(),
 )
 api = register_urls(api)
 urls = api.urls
@@ -121,7 +113,24 @@ def generic_exception_handler(request, err):
             "message": f"{err.__class__.__name__}: {err}",
             "errors": [
                 "".join(format_exception(err)),
+                # or send simpler parent-only traceback:
+                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
             ],
         },
         status=status,
     )
+
+
+# import orjson
+# from ninja.renderers import BaseRenderer
+# class ORJSONRenderer(BaseRenderer):
+#     media_type = "application/json"
+#     def render(self, request, data, *, response_status):
+#         return {
+#             "success": True,
+#             "errors": [],
+#             "result": data,
+#             "stdout": ansi_to_html(stdout.getvalue().strip()),
+#             "stderr": ansi_to_html(stderr.getvalue().strip()),
+#         }
+#         return orjson.dumps(data)

archivebox/api/v1_auth.py

@@ -4,7 +4,7 @@ from django.http import HttpRequest
 
 from ninja import Router, Schema
 
-from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token, HeaderTokenAuth, BearerTokenAuth
+from archivebox.api.auth import auth_using_token, auth_using_password, get_or_create_api_token
 
 
 router = Router(tags=["Authentication"], auth=None)
@@ -63,44 +63,3 @@ def check_api_token(request: HttpRequest, token_data: TokenAuthSchema):
         return {"success": True, "user_id": str(user.pk)}
 
     return {"success": False, "user_id": None}
-
-
-@router.post(
-    "/browser_session",
-    auth=[HeaderTokenAuth(), BearerTokenAuth()],
-    summary="Exchange an administrator API key for a normal browser session",
-)
-def browser_session(request: HttpRequest):
-    from django.conf import settings
-    from django.contrib.auth import login
-    from django.http import JsonResponse
-    from django.utils import timezone
-    from ninja.errors import HttpError
-    from archivebox.core.routes_util import get_admin_base_url
-
-    user = request.auth
-    if not user.is_active or not user.is_superuser:
-        raise HttpError(403, "An active administrator is required")
-    login(request, user, backend="django.contrib.auth.backends.ModelBackend")
-    # A browser session must not outlive the key used to establish it.
-    token = request._api_token
-    lifetime = settings.SESSION_COOKIE_AGE
-    if token.expires:
-        lifetime = min(lifetime, max(0, int((token.expires - timezone.now()).total_seconds())))
-    request.session.set_expiry(lifetime)
-    request.session.save()
-    response = JsonResponse(
-        {
-            "admin_url": get_admin_base_url(request=request).rstrip("/") + "/admin/",
-            "cookie": {
-                "name": settings.SESSION_COOKIE_NAME,
-                "value": request.session.session_key,
-                "expires": request.session.get_expiry_date().timestamp(),
-                "secure": settings.SESSION_COOKIE_SECURE,
-            },
-        },
-    )
-    # The native client installs a host-only HttpOnly cookie on the admin origin;
-    # API and admin hosts can differ, so a Set-Cookie on the API host is insufficient.
-    response["Cache-Control"] = "no-store"
-    return response

archivebox/api/v1_cli.py

@@ -8,13 +8,14 @@ from enum import Enum
 from django.http import HttpRequest
 
 from ninja import Router, Schema
-from ninja.errors import HttpError
 from pydantic import Field
 
 from archivebox.misc.util import ansi_to_html
-from archivebox.core.models import SnapshotQuerySet
+from archivebox.config.common import get_config
 
 
+# from .auth import API_AUTH_METHODS
+
 # router for API that exposes archivebox cli subcommands as REST endpoints
 router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 
@@ -22,7 +23,6 @@ router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 # Schemas
 
 JSONType = list[Any] | dict[str, Any] | bool | int | str | None
-FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
 
 
 class CLICommandResponseSchema(Schema):
@@ -34,11 +34,26 @@ class CLICommandResponseSchema(Schema):
     stderr: str
 
 
-FilterTypeChoices = Enum(
-    "FilterTypeChoices",
-    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
-    type=str,
-)
+class FilterTypeChoices(str, Enum):
+    exact = "exact"
+    substring = "substring"
+    regex = "regex"
+    domain = "domain"
+    tag = "tag"
+    timestamp = "timestamp"
+
+
+class StatusChoices(str, Enum):
+    indexed = "indexed"
+    archived = "archived"
+    unarchived = "unarchived"
+    present = "present"
+    valid = "valid"
+    invalid = "invalid"
+    duplicate = "duplicate"
+    orphaned = "orphaned"
+    corrupted = "corrupted"
+    unrecognized = "unrecognized"
 
 
 class AddCommandSchema(Schema):
@@ -52,34 +67,19 @@ class AddCommandSchema(Schema):
     snapshot_max_size: int = 0
     parser: str = "auto"
     plugins: str = ""
-    persona: str = "Default"
-    only_new: bool | None = None
-    update: bool = False
+    update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
     overwrite: bool = False
     index_only: bool = False
 
 
-class SnapshotFilterCommandSchema(Schema):
-    after: float | None = 0
-    before: float | None = None
-    filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
-    status: str | None = None
-    url__icontains: str | None = None
-    url__istartswith: str | None = None
-    tag: str | None = None
-    crawl_id: str | None = None
-    limit: int | None = None
-    sort: str | None = None
-    search: str | None = None
-
-
-class UpdateCommandSchema(SnapshotFilterCommandSchema):
+class UpdateCommandSchema(Schema):
     resume: str | None = None
+    after: float | None = 0
+    before: float | None = 999999999999999
+    filter_type: str | None = FilterTypeChoices.substring
+    filter_patterns: list[str] | None = ["https://example.com"]
     batch_size: int = 100
     continuous: bool = False
-    index_only: bool = False
-    migrate_only: bool = False
 
 
 class ScheduleCommandSchema(Schema):
@@ -92,58 +92,37 @@ class ScheduleCommandSchema(Schema):
     every: str | None = None
     tag: str = ""
     depth: int = 0
-    only_new: bool | None = None
-    update: bool = False
     overwrite: bool = False
+    update: bool = Field(default_factory=lambda: not get_config().ONLY_NEW)
     clear: bool = False
 
 
-class ListCommandSchema(SnapshotFilterCommandSchema):
+class ListCommandSchema(Schema):
+    filter_patterns: list[str] | None = ["https://example.com"]
+    filter_type: str = FilterTypeChoices.substring
+    status: StatusChoices = StatusChoices.indexed
+    after: float | None = 0
+    before: float | None = 999999999999999
+    sort: str = "bookmarked_at"
     as_json: bool = True
     as_html: bool = False
     as_csv: str | None = "timestamp,url"
     with_headers: bool = False
 
 
-class RemoveCommandSchema(SnapshotFilterCommandSchema):
+class RemoveCommandSchema(Schema):
+    after: float | None = 0
+    before: float | None = 999999999999999
     filter_type: str = FilterTypeChoices.exact
-    timeout: float = 60.0
-
-
-def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
-    kwargs = args.dict()
-    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
-    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
-    return kwargs
+    filter_patterns: list[str] | None = ["https://example.com"]
 
 
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
     from archivebox.cli.archivebox_add import add
-    from archivebox.misc.util import validate_url
 
-    config_overrides: dict[str, object] = {}
-    if args.only_new is not None:
-        config_overrides["ONLY_NEW"] = bool(args.only_new)
-    if args.update or args.overwrite:
-        config_overrides["ONLY_NEW"] = False
-    submitted_urls: str | list[str] = args.urls
-    if len(args.urls) == 1:
-        try:
-            validate_url(args.urls[0])
-        except ValueError:
-            submitted_urls = args.urls[0]
-    else:
-        submitted_urls = []
-        for url in args.urls:
-            try:
-                submitted_urls.append(validate_url(url))
-            except ValueError:
-                continue
-        if not submitted_urls:
-            raise HttpError(400, "No valid URLs were submitted")
     crawl, snapshots = add(
-        urls=submitted_urls,
+        urls=args.urls,
         snapshot_ids=args.snapshot_ids,
         tag=args.tag,
         depth=args.depth,
@@ -151,13 +130,13 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         crawl_max_size=args.crawl_max_size,
         crawl_timeout=args.crawl_timeout,
         snapshot_max_size=args.snapshot_max_size,
+        update=args.update,
         index_only=args.index_only,
+        overwrite=args.overwrite,
         plugins=args.plugins,
-        persona=args.persona,
         parser=args.parser,
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,
-        config=config_overrides or None,
     )
 
     snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots.values_list("id", flat=True)]
@@ -165,10 +144,10 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         "crawl_id": str(crawl.id),
         "num_snapshots": len(snapshot_ids),
         "snapshot_ids": snapshot_ids,
-        "queued_urls": args.urls if isinstance(submitted_urls, str) else submitted_urls,
+        "queued_urls": args.urls,
     }
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
 
     return {
         "success": True,
@@ -182,39 +161,24 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
 
 @router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
 def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
-    from archivebox.core.snapshot_status import normalize_snapshot_status
+    from archivebox.cli.archivebox_update import update
 
-    try:
-        status = normalize_snapshot_status(args.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-
-    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
-    update_kwargs["status"] = status
-    update_kwargs["stop_daemon_stack"] = False
-
-    is_filtered_update = any(
-        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
+    result = update(
+        filter_patterns=args.filter_patterns or [],
+        filter_type=args.filter_type or FilterTypeChoices.substring,
+        after=args.after,
+        before=args.before,
+        resume=args.resume,
+        batch_size=args.batch_size,
+        continuous=args.continuous,
+        stop_daemon_stack=False,
     )
-    matched_snapshot_ids = []
-    if is_filtered_update:
-        matched_snapshot_ids = [
-            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
-        ]
-
-    update(**update_kwargs)
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
     return {
         "success": True,
         "errors": [],
-        "result": {
-            "matched_count": len(matched_snapshot_ids),
-            "snapshot_ids": matched_snapshot_ids,
-        }
-        if is_filtered_update
-        else None,
+        "result": result,
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
         "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
     }
@@ -224,11 +188,6 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
 def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
     from archivebox.cli.archivebox_schedule import schedule
 
-    config_overrides: dict[str, object] = {}
-    if args.only_new is not None:
-        config_overrides["ONLY_NEW"] = bool(args.only_new)
-    if args.update or args.overwrite:
-        config_overrides["ONLY_NEW"] = False
     result = schedule(
         import_path=args.import_path,
         add=args.add,
@@ -240,11 +199,12 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
         every=args.every,
         tag=args.tag,
         depth=args.depth,
-        config=config_overrides or None,
+        overwrite=args.overwrite,
+        update=args.update,
     )
 
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
     return {
         "success": True,
         "errors": [],
@@ -257,38 +217,32 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
 
 @router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
 def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
+    from archivebox.cli.archivebox_search import search
 
-    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
-    as_json = search_kwargs.pop("as_json")
-    as_html = search_kwargs.pop("as_html")
-    as_csv = search_kwargs.pop("as_csv")
-    with_headers = search_kwargs.pop("with_headers")
-    try:
-        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
+    result = search(
+        filter_patterns=args.filter_patterns,
+        filter_type=args.filter_type,
+        status=args.status,
+        after=args.after,
+        before=args.before,
+        sort=args.sort,
+        csv=args.as_csv,
+        json=args.as_json,
+        html=args.as_html,
+        with_headers=args.with_headers,
+    )
 
     result_format = "txt"
-    if as_json:
+    if args.as_json:
         result_format = "json"
-        result = [
-            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
-            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
-        ]
-    elif as_html:
+        result = json.loads(result)
+    elif args.as_html:
         result_format = "html"
-        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
-    elif as_csv:
+    elif args.as_csv:
         result_format = "csv"
-        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
-        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
-        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
-    else:
-        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
 
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
     return {
         "success": True,
         "errors": [],
@@ -302,26 +256,37 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
 @router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
 def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
     from archivebox.cli.archivebox_remove import remove
+    from archivebox.cli.archivebox_search import get_snapshots
     from archivebox.core.models import Snapshot
 
-    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
-    timeout_arg = remove_kwargs.pop("timeout")
-    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
-    try:
-        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
+    filter_patterns = args.filter_patterns or []
+    snapshots_to_remove = get_snapshots(
+        filter_patterns=filter_patterns,
+        filter_type=args.filter_type,
+        after=args.after,
+        before=args.before,
+    )
+    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
 
-    result = remove(
+    remove(
         yes=True,  # no way to interactively ask for confirmation via API, so we force yes
         snapshots=snapshots_to_remove,
-        timeout=timeout,
+        before=args.before,
+        after=args.after,
+        filter_type=args.filter_type,
+        filter_patterns=filter_patterns,
     )
-    stdout = request.__dict__.get("stdout")
-    stderr = request.__dict__.get("stderr")
+
+    result = {
+        "removed_count": len(removed_snapshot_ids),
+        "removed_snapshot_ids": removed_snapshot_ids,
+        "remaining_snapshots": Snapshot.objects.count(),
+    }
+    stdout = getattr(request, "stdout", None)
+    stderr = getattr(request, "stderr", None)
     return {
-        "success": bool(result["success"]),
-        "errors": [str(result["error"])] if result["error"] else [],
+        "success": True,
+        "errors": [],
         "result": result,
         "result_format": "json",
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",

archivebox/api/v1_core.py

@@ -4,14 +4,13 @@ import math
 import json
 import mimetypes
 import re
-from html import unescape
 from collections import defaultdict
 from pathlib import Path, PurePosixPath
 from uuid import UUID
 from typing import Union, Any, Annotated
 from datetime import datetime, time
 
-from django.db import IntegrityError
+from django.db import transaction
 from django.db.models import Model, Q
 from django.http import HttpRequest, HttpResponse
 from django.http.multipartparser import MultiPartParser, MultiPartParserError
@@ -30,11 +29,11 @@ from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot, ArchiveResult, Tag
 from archivebox.core.permissions import public_snapshots_queryset
-from archivebox.api.auth import authenticated_user_from_request
+from archivebox.api.auth import auth_using_token
 from archivebox.config.common import get_config
-from archivebox.core.routes_util import build_snapshot_detail_url, build_web_url
-from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
-from archivebox.core.tag_util import (
+from archivebox.core.host_utils import build_web_url
+from archivebox.misc.util import validate_url_length
+from archivebox.core.tag_utils import (
     add_snapshot_counts,
     build_tag_cards,
     delete_tag as delete_tag_record,
@@ -50,16 +49,12 @@ from archivebox.core.tag_util import (
     rename_tag as rename_tag_record,
 )
 from archivebox.crawls.models import Crawl
-from archivebox.crawls.locks import crawl_lifecycle_lock
-from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref
-from archivebox.search.config import get_search_mode, get_search_mode_backend
-from archivebox.search.query import apply_snapshot_search
-from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
+from archivebox.api.v1_crawls import CrawlSchema
 
 
 router = Router(tags=["Core Models"])
 
-ARCHIVERESULT_UPLOAD_HOOK_NAME = Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME
+ARCHIVERESULT_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload"
 ARCHIVERESULT_UPLOAD_PLUGIN_RE = re.compile(r"^[A-Za-z0-9_.-]{1,32}$")
 
 
@@ -82,7 +77,7 @@ class CustomPagination(PaginationBase):
     def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
         limit = min(pagination.limit, 500)
         offset = pagination.offset or (pagination.page * limit)
-        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
+        total = queryset.count()
         total_pages = math.ceil(total / limit)
         current_page = math.ceil(offset / (limit + 1))
         items = queryset[offset : offset + limit]
@@ -211,59 +206,40 @@ class ArchiveResultFilterSchema(FilterSchema):
     created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
 
 
-def _with_archiveresult_processes(queryset):
-    # cmd/pwd/version delegate to Process and Binary. Without this join a full
-    # capture's metadata response performs two extra indexed lookups per hook
-    # (roughly 100 queries), despite the initial Snapshot UUID lookup being fast.
-    # Logs and the hydrated hook environment are not response fields; loading
-    # those potentially large values would replace round trips with excess I/O.
-    return queryset.select_related("process__binary").defer("process__stdout", "process__stderr", "process__env")
-
-
 @router.get("/archiveresults", response=list[ArchiveResultSchema], url_name="get_archiveresult")
 @paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
     """List all ArchiveResult entries matching these filters."""
-    queryset = filters.filter(
-        _with_archiveresult_processes(ArchiveResult.objects.all()).prefetch_related("snapshot__crawl__created_by", "snapshot__tags"),
-    )
-    if filters.search or filters.snapshot_tag:
-        return queryset.distinct()
-    return queryset
+    return filters.filter(ArchiveResult.objects.all()).distinct()
 
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
     raw_ref = str(ref or "").strip()
+    query = Q(**{f"{field_name}__startswith": raw_ref})
+    if raw_ref:
+        query |= Q(**{f"{field_name}__icontains": raw_ref})
     try:
         parsed_uuid = UUID(raw_ref)
     except (TypeError, ValueError):
-        query = Q(**{f"{field_name}__startswith": raw_ref})
-        if raw_ref:
-            query |= Q(**{f"{field_name}__icontains": raw_ref})
         normalized_ref = raw_ref.replace("-", "")
         if normalized_ref and normalized_ref != raw_ref:
             query |= Q(**{f"{field_name}__startswith": normalized_ref})
             query |= Q(**{f"{field_name}__icontains": normalized_ref})
-        return query
-    # An exact UUID cannot need a substring fallback. OR-ing it with LIKE
-    # prevents an indexed lookup and made Cabbage metadata reads take seconds.
-    return Q(**{field_name: parsed_uuid})
+    else:
+        query |= Q(**{field_name: parsed_uuid})
+        query |= Q(**{f"{field_name}__startswith": parsed_uuid.hex})
+    return query
 
 
 @router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
 def get_archiveresult(request: HttpRequest, archiveresult_id: str):
     """Get a specific ArchiveResult by id."""
-    return (
-        _with_archiveresult_processes(ArchiveResult.objects.all())
-        .select_related("snapshot__crawl__created_by")
-        .prefetch_related("snapshot__tags")
-        .get(_uuid_ref_query("id", archiveresult_id))
-    )
+    return ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
 
 
 def _normalize_uploaded_archiveresult_plugin(plugin: str) -> str:
     normalized = str(plugin or "").strip().strip("/")
-    if normalized in (".", "..") or not ARCHIVERESULT_UPLOAD_PLUGIN_RE.fullmatch(normalized):
+    if not ARCHIVERESULT_UPLOAD_PLUGIN_RE.fullmatch(normalized):
         raise HttpError(400, "Invalid ArchiveResult plugin name")
     return normalized
 
@@ -295,7 +271,7 @@ def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any]
 
 
 def _get_archiveresult_upload_data(request: HttpRequest):
-    cached = request.__dict__.get("_archiveresult_upload_data")
+    cached = getattr(request, "_archiveresult_upload_data", None)
     if cached is not None:
         return cached
 
@@ -358,20 +334,25 @@ def _parse_archiveresult_upload_int(value: str, field_name: str, *, default: int
 
 
 def _summarize_archiveresult_output_files(output_files: dict[str, dict[str, Any]]) -> tuple[int, str]:
-    from abx_dl.output_files import OutputManifest
-
-    manifest = OutputManifest.from_value(output_files)
-    return manifest.total_size, ",".join(manifest.mimetypes)
+    mime_sizes: dict[str, int] = defaultdict(int)
+    total_size = 0
+    for metadata in output_files.values():
+        if not isinstance(metadata, dict):
+            continue
+        try:
+            size = max(int(metadata.get("size") or 0), 0)
+        except (TypeError, ValueError):
+            size = 0
+        mime_type = str(metadata.get("mimetype") or "").strip()
+        total_size += size
+        if mime_type and size:
+            mime_sizes[mime_type] += size
+    output_mimetypes = ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
+    return total_size, output_mimetypes
 
 
 def _get_snapshot_by_ref(snapshot_id: str):
     queryset = Snapshot.objects.select_related("crawl__created_by")
-    try:
-        full_id = UUID(snapshot_id)
-    except (TypeError, ValueError):
-        pass
-    else:
-        return queryset.get(pk=full_id)
     try:
         return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
@@ -400,6 +381,13 @@ def _queue_archiveresult_snapshot_maintenance(snapshot: Snapshot) -> None:
     snapshot.safe_update(updates, refresh=False)
 
 
+def _merge_archiveresult_output_file_maps(results: list[ArchiveResult]) -> dict[str, dict[str, Any]]:
+    output_files: dict[str, dict[str, Any]] = {}
+    for result in results:
+        output_files.update(result.output_file_map())
+    return output_files
+
+
 def _write_archiveresult_files(
     request: HttpRequest,
     snapshot: Snapshot,
@@ -408,19 +396,18 @@ def _write_archiveresult_files(
     existing_output_files: dict[str, dict[str, Any]] | None = None,
     allow_empty: bool = False,
 ) -> dict[str, dict[str, Any]]:
-    plugin_name = _normalize_uploaded_archiveresult_plugin(plugin_name)
     files = _get_archiveresult_upload_files(request, allow_empty=allow_empty)
     output_paths = _get_archiveresult_upload_form_values(request, "output_paths", "output_path")
     mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
     chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
 
-    output_files = dict(existing_output_files or {})
-    if not files:
-        return output_files
-
     snapshot_dir = snapshot.output_dir
     plugin_dir = snapshot_dir / plugin_name
     storage = FileSystemStorage(location=str(plugin_dir))
+    output_files = dict(existing_output_files or {})
+
+    if not files:
+        return output_files
 
     if chunk_output_path:
         if len(files) != 1:
@@ -479,7 +466,12 @@ def _write_archiveresult_files(
             )
 
         guessed_mime = mimetypes.guess_type(relative_output_path)[0]
-        output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
+        output_mime_type = (
+            (mime_types[0] if mime_types else "")
+            or getattr(uploaded_file, "content_type", None)
+            or guessed_mime
+            or "application/octet-stream"
+        )
         output_files[relative_output_path] = {
             "extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
             "mimetype": output_mime_type,
@@ -507,7 +499,7 @@ def _write_archiveresult_files(
         guessed_mime = mimetypes.guess_type(saved_output_path)[0]
         output_mime_type = (
             (mime_types[index] if index < len(mime_types) else "")
-            or uploaded_file.content_type
+            or getattr(uploaded_file, "content_type", None)
             or guessed_mime
             or "application/octet-stream"
         )
@@ -540,53 +532,67 @@ def create_archiveresult(
     normalized_status = ArchiveResult.normalize_status(status)
     parsed_output_json = _parse_archiveresult_output_json(output_json)
     hook = hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME
-    result_lookup = {
-        "snapshot": snapshot,
-        "plugin": plugin_name,
-        "hook_name": hook,
-    }
-    uploaded_output_files = _write_archiveresult_files(
+    matching_results = list(
+        ArchiveResult.objects.filter(
+            snapshot=snapshot,
+            plugin=plugin_name,
+            hook_name=hook,
+        ).order_by("created_at", "id"),
+    )
+    existing_result = matching_results[0] if matching_results else None
+    existing_output_files = _merge_archiveresult_output_file_maps(matching_results)
+    output_files = _write_archiveresult_files(
         request,
         snapshot,
         plugin_name,
+        existing_output_files=existing_output_files,
         allow_empty=True,
     )
-    result = ArchiveResult.objects.filter(**result_lookup).first()
-    for _attempt in range(3):
-        output_files = {
-            **(result.output_file_map() if result else {}),
-            **uploaded_output_files,
-        }
+    now = timezone.now()
+
+    with transaction.atomic():
+        matching_results = list(
+            ArchiveResult.objects.filter(
+                snapshot=snapshot,
+                plugin=plugin_name,
+                hook_name=hook,
+            ).order_by("created_at", "id"),
+        )
+        if matching_results:
+            existing_result = matching_results[0]
+            output_files = {
+                **_merge_archiveresult_output_file_maps(matching_results),
+                **output_files,
+            }
+            duplicate_ids = [result.id for result in matching_results[1:]]
+            if duplicate_ids:
+                ArchiveResult.objects.filter(id__in=duplicate_ids).delete()
+            result = existing_result
+        else:
+            existing_result = None
+            result = ArchiveResult(
+                snapshot=snapshot,
+                plugin=plugin_name,
+                hook_name=hook,
+            )
+
+        if (
+            existing_result
+            and normalized_status == ArchiveResult.StatusChoices.STARTED
+            and existing_result.status != ArchiveResult.StatusChoices.STARTED
+        ):
+            normalized_status = existing_result.status
         output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
         output_file_paths = list(output_files.keys())
-        result_status = normalized_status
-        if result and result_status == ArchiveResult.StatusChoices.STARTED and result.status != ArchiveResult.StatusChoices.STARTED:
-            result_status = result.status
-        now = timezone.now()
-        values = {
-            "status": result_status,
-            "output_str": output_str or (output_file_paths[0] if output_file_paths else ""),
-            "output_json": parsed_output_json,
-            "output_files": output_files,
-            "output_size": output_size,
-            "output_mimetypes": output_mimetypes,
-            "start_ts": result.start_ts or now if result else now,
-            "end_ts": now,
-        }
-        if result:
-            if result.safe_update(values):
-                break
-            continue
-        result, created = ArchiveResult.get_or_create_by_hook(
-            snapshot,
-            plugin_name,
-            hook,
-            defaults=values,
-        )
-        if created:
-            break
-    else:
-        raise HttpError(409, "ArchiveResult changed while upload metadata was being updated")
+        result.status = normalized_status
+        result.output_str = output_str or (output_file_paths[0] if output_file_paths else "")
+        result.output_json = parsed_output_json
+        result.output_files = output_files
+        result.output_size = output_size
+        result.output_mimetypes = output_mimetypes
+        result.start_ts = result.start_ts or now
+        result.end_ts = now
+        result.save()
 
     if result.status != ArchiveResult.StatusChoices.STARTED:
         _queue_archiveresult_snapshot_maintenance(snapshot)
@@ -600,46 +606,44 @@ def patch_archiveresult(
 ):
     """Append or replace files on an existing ArchiveResult."""
     result = ArchiveResult.objects.select_related("snapshot__crawl__created_by").get(_uuid_ref_query("id", archiveresult_id))
-    uploaded_output_files = _write_archiveresult_files(
+    output_files = _write_archiveresult_files(
         request,
         result.snapshot,
         result.plugin,
+        existing_output_files=result.output_file_map(),
     )
+    latest_result = ArchiveResult.objects.only("output_files", "status").get(pk=result.pk)
+    output_files = {
+        **latest_result.output_file_map(),
+        **output_files,
+    }
+    output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
+
+    update_fields = ["output_files", "output_size", "output_mimetypes", "end_ts", "modified_at"]
+    result.output_files = output_files
+    result.output_size = output_size
+    result.output_mimetypes = output_mimetypes
+    result.end_ts = timezone.now()
     output_str = _get_archiveresult_upload_form_value(request, "output_str")
     status = _get_archiveresult_upload_form_value(request, "status")
     output_json = _get_archiveresult_upload_form_value(request, "output_json")
-    parsed_output_json = _parse_archiveresult_output_json(output_json) if output_json else None
-
-    if not ArchiveResult.output_files_upload_complete(uploaded_output_files):
-        result.output_files = {**result.output_file_map(), **uploaded_output_files}
-        result.output_size, result.output_mimetypes = _summarize_archiveresult_output_files(result.output_files)
-        return result
-
-    for _attempt in range(3):
-        output_files = {**result.output_file_map(), **uploaded_output_files}
-        output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
-        values: dict[str, Any] = {
-            "output_files": output_files,
-            "output_size": output_size,
-            "output_mimetypes": output_mimetypes,
-            "end_ts": timezone.now(),
-        }
-        if output_str:
-            values["output_str"] = output_str
-        if status:
-            normalized_status = ArchiveResult.normalize_status(status)
-            if normalized_status == ArchiveResult.StatusChoices.STARTED and result.status != ArchiveResult.StatusChoices.STARTED:
-                normalized_status = result.status
-            values["status"] = normalized_status
-        elif result.status == ArchiveResult.StatusChoices.QUEUED:
-            values["status"] = ArchiveResult.StatusChoices.SUCCEEDED
-        if output_json:
-            values["output_json"] = parsed_output_json
-        if result.safe_update(values):
-            break
-    else:
-        raise HttpError(409, "ArchiveResult changed while upload metadata was being updated")
+    if output_str:
+        result.output_str = output_str
+        update_fields.append("output_str")
+    if status:
+        normalized_status = ArchiveResult.normalize_status(status)
+        if normalized_status == ArchiveResult.StatusChoices.STARTED and latest_result.status != ArchiveResult.StatusChoices.STARTED:
+            normalized_status = latest_result.status
+        result.status = normalized_status
+        update_fields.append("status")
+    elif latest_result.status == ArchiveResult.StatusChoices.QUEUED and ArchiveResult.output_files_upload_complete(output_files):
+        result.status = ArchiveResult.StatusChoices.SUCCEEDED
+        update_fields.append("status")
+    if output_json:
+        result.output_json = _parse_archiveresult_output_json(output_json)
+        update_fields.append("output_json")
 
+    result.save(update_fields=update_fields)
     if result.status != ArchiveResult.StatusChoices.STARTED:
         _queue_archiveresult_snapshot_maintenance(result.snapshot)
 
@@ -652,8 +656,6 @@ def patch_archiveresult(
 class SnapshotSchema(Schema):
     TYPE: str = "core.models.Snapshot"
     id: UUID
-    crawl_id: UUID
-    persona: str | None
     created_by_id: str
     created_by_username: str
     created_at: datetime
@@ -672,17 +674,6 @@ class SnapshotSchema(Schema):
     num_archiveresults: int
     archiveresults: list[MinimalArchiveResultSchema]
 
-    @staticmethod
-    def resolve_archive_path(obj):
-        return obj.archive_path_from_db
-
-    @staticmethod
-    def resolve_persona(obj):
-        # A deduplicated submission can reuse a capture from a different
-        # persona. Report its actual DB-backed owner, not the new request's choice.
-        persona = obj.crawl.resolve_persona()
-        return persona.name if persona else None
-
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by.pk)
@@ -705,14 +696,12 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_num_archiveresults(obj, context):
-        # This reverse FK contains each result once. DISTINCT would materialize
-        # wide JSON output columns just to count rows through snapshot_id's index.
-        return obj.archiveresult_set.count()
+        return obj.archiveresult_set.all().distinct().count()
 
     @staticmethod
     def resolve_archiveresults(obj, context):
-        if bool(context["request"].__dict__.get("with_archiveresults", False)):
-            return _with_archiveresult_processes(obj.archiveresult_set.all())
+        if bool(getattr(context["request"], "with_archiveresults", False)):
+            return obj.archiveresult_set.all().distinct()
         return ArchiveResult.objects.none()
 
 
@@ -737,7 +726,6 @@ class SnapshotDeleteResponseSchema(Schema):
     snapshot_id: str
     crawl_id: str
     deleted_count: int
-    queued_count: int
 
 
 def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
@@ -798,8 +786,7 @@ def _filter_snapshots_for_rss(
     )
     crawl_id = crawl_id.strip()
     if crawl_id:
-        matching_crawl_pks = list(filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).values_list("pk", flat=True)[:100])
-        queryset = queryset.filter(crawl_id__in=matching_crawl_pks)
+        queryset = queryset.filter(crawl__id__icontains=crawl_id)
 
     created_by = created_by.strip()
     if created_by:
@@ -838,12 +825,12 @@ def _snapshots_rss_response(
     )
 
     for snapshot in snapshots:
-        archived_url = build_snapshot_detail_url(snapshot.archive_path_from_db, request=request)
+        archived_url = build_web_url(f"/{snapshot.archive_path_from_db}", request=request)
         tags = [tag.name for tag in snapshot.tags.all()]
         crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
         description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
         feed.add_item(
-            title=unescape(snapshot.title or snapshot.url),
+            title=snapshot.title or snapshot.url,
             link=archived_url or web_base_url,
             description=description,
             unique_id=str(snapshot.id),
@@ -858,7 +845,7 @@ def _snapshots_rss_response(
 
 
 class SnapshotFilterSchema(FilterSchema):
-    id: Annotated[str | None, FilterLookup(["id__istartswith", "id__iendswith", "timestamp__startswith"])] = None
+    id: Annotated[str | None, FilterLookup(["id__icontains", "timestamp__startswith"])] = None
     created_by_id: Annotated[str | None, FilterLookup("crawl__created_by_id")] = None
     created_by_username: Annotated[str | None, FilterLookup("crawl__created_by__username__icontains")] = None
     created_at__gte: Annotated[datetime | None, FilterLookup("created_at__gte")] = None
@@ -867,9 +854,10 @@ class SnapshotFilterSchema(FilterSchema):
     modified_at: Annotated[datetime | None, FilterLookup("modified_at")] = None
     modified_at__gte: Annotated[datetime | None, FilterLookup("modified_at__gte")] = None
     modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
-    search: str | None = None
-    search_mode: str | None = None
-    status: str | None = None
+    search: Annotated[
+        str | None,
+        FilterLookup(["url__icontains", "title__icontains", "tags__name__icontains", "id__icontains", "timestamp__startswith"]),
+    ] = None
     url: Annotated[str | None, FilterLookup("url")] = None
     tag: Annotated[str | None, FilterLookup("tags__name")] = None
     title: Annotated[str | None, FilterLookup("title__icontains")] = None
@@ -877,44 +865,14 @@ class SnapshotFilterSchema(FilterSchema):
     bookmarked_at__gte: Annotated[datetime | None, FilterLookup("bookmarked_at__gte")] = None
     bookmarked_at__lt: Annotated[datetime | None, FilterLookup("bookmarked_at__lt")] = None
 
-    def filter_search(self, value: str | None) -> Q:
-        return Q()
-
-    def filter_search_mode(self, value: str | None) -> Q:
-        return Q()
-
-    def filter_status(self, value: str | None) -> Q:
-        return Q()
-
 
 @router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
 @paginate(CustomPagination)
 def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
     """List all Snapshot entries matching these filters."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    try:
-        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
-    queryset = filters.filter(queryset).distinct()
-    query = (filters.search or "").strip()
-    if not query:
-        return queryset
-
-    runtime_config = request.archivebox_config
-    search_mode = get_search_mode(filters.search_mode, config=runtime_config)
-    try:
-        return apply_snapshot_search(
-            queryset,
-            query,
-            search_mode=search_mode,
-            config=runtime_config,
-            include_id_matches=True,
-        )
-    except Exception:
-        if get_search_mode_backend(search_mode, config=runtime_config):
-            return queryset.none()
-        return apply_snapshot_search(queryset, query, search_mode="meta", config=runtime_config, include_id_matches=True)
+    queryset = Snapshot.objects.all()
+    return filters.filter(queryset).distinct()
 
 
 @router.get("/snapshots.rss", url_name="get_snapshots_rss")
@@ -939,16 +897,18 @@ def get_snapshots_rss(
 def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
     """Get a specific Snapshot by id."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    return _get_snapshot_by_ref(snapshot_id)
+    queryset = Snapshot.objects.all()
+    try:
+        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
+    except Snapshot.DoesNotExist:
+        return queryset.get(_uuid_ref_query("id", snapshot_id))
 
 
 @router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
 def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     tags = normalize_tag_list(data.tags)
-    try:
-        status = normalize_snapshot_status(data.status)
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
+    if data.status is not None and data.status not in Snapshot.StatusChoices.values:
+        raise HttpError(400, f"Invalid status: {data.status}")
     if not data.url.strip():
         raise HttpError(400, "URL is required")
     try:
@@ -959,7 +919,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         raise HttpError(400, "depth must be between 0 and 4")
 
     if data.crawl_id:
-        crawl = get_crawl_by_ref(data.crawl_id)
+        crawl = Crawl.objects.get(id__icontains=data.crawl_id)
         crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
         tags = tags or crawl_tags
     else:
@@ -972,51 +932,39 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    # Browser uploads must not wait behind the runner's crawl-wide lifecycle
-    # lock. The unique insert recovery and CAS update below are the request-side
-    # coordination boundary for this idempotent metadata sync.
-    snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-    if snapshot is None and bool(crawl.get_current_config().get("ONLY_NEW", True)):
-        # Uploading browser metadata is part of the same submission as /add.
-        # It must honor the runner's deduplication policy, even if it arrives
-        # before the runner has processed that crawl. Report the actual owner
-        # so clients never mistake a reused capture for one they just created.
-        snapshot = Snapshot.objects.filter(url=data.url).exclude(status=Snapshot.DELETING_STATE).order_by("-created_at").first()
-        if snapshot is not None:
-            setattr(request, "with_archiveresults", False)
-            return snapshot
-    if snapshot is None:
-        try:
-            snapshot = Snapshot.objects.create(
-                url=data.url,
-                crawl=crawl,
-                depth=data.depth,
-                title=data.title,
-                timestamp=str(timezone.now().timestamp()),
-                status=status or Snapshot.StatusChoices.QUEUED,
-                retry_at=timezone.now(),
-            )
-        except IntegrityError:
-            snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-            if snapshot is None:
-                raise
+    snapshot_defaults = {
+        "depth": data.depth,
+        "title": data.title,
+        "timestamp": str(timezone.now().timestamp()),
+        "status": data.status or Snapshot.StatusChoices.QUEUED,
+        "retry_at": timezone.now(),
+    }
+    snapshot, _ = Snapshot.objects.get_or_create(
+        url=data.url,
+        crawl=crawl,
+        defaults=snapshot_defaults,
+    )
 
-    for _attempt in range(3):
-        updates: dict[str, Any] = {}
-        if data.title is not None and snapshot.title != data.title:
-            updates["title"] = data.title
-        if status is not None and snapshot.status != status:
-            updates["status"] = status
-        if not updates or snapshot.safe_update(updates, extra_filter={"modified_at": snapshot.modified_at}):
-            break
-    else:
-        raise HttpError(409, "Snapshot changed while metadata was being updated")
+    update_fields: list[str] = []
+    if data.title is not None and snapshot.title != data.title:
+        snapshot.title = data.title
+        update_fields.append("title")
+    if data.status is not None and snapshot.status != data.status:
+        if data.status not in Snapshot.StatusChoices.values:
+            raise HttpError(400, f"Invalid status: {data.status}")
+        snapshot.status = data.status
+        update_fields.append("status")
+    if update_fields:
+        update_fields.append("modified_at")
+        snapshot.save(update_fields=update_fields)
 
     if tags:
-        snapshot.save_tags(
-            tags,
-            created_by=request.user if isinstance(request.user, User) else None,
-        )
+        snapshot.save_tags(tags)
+
+    try:
+        snapshot.ensure_crawl_symlink()
+    except Exception:
+        pass
 
     setattr(request, "with_archiveresults", False)
     return snapshot
@@ -1025,8 +973,10 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
 @router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
 def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
     """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    snapshot = _get_snapshot_by_ref(snapshot_id)
-    crawl_id = str(snapshot.crawl_id)
+    try:
+        snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
+    except Snapshot.DoesNotExist:
+        snapshot = Snapshot.objects.get(Q(id__icontains=snapshot_id))
 
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
@@ -1048,54 +998,40 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
             return snapshot
         raise HttpError(400, f"Invalid action: {action}")
 
-    with crawl_lifecycle_lock(crawl_id):
-        snapshot = _get_snapshot_by_ref(snapshot_id)
-        if "status" in payload:
-            try:
-                snapshot.status = normalize_snapshot_status(payload["status"])
-            except ValueError as err:
-                raise HttpError(400, str(err)) from err
-            if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
-                snapshot.retry_at = None
-            update_fields.append("status")
+    if "status" in payload:
+        if payload["status"] not in Snapshot.StatusChoices.values:
+            raise HttpError(400, f"Invalid status: {payload['status']}")
+        snapshot.status = payload["status"]
+        if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
+            snapshot.retry_at = None
+        update_fields.append("status")
 
-        if "retry_at" in payload:
-            snapshot.retry_at = payload["retry_at"]
-            update_fields.append("retry_at")
+    if "retry_at" in payload:
+        snapshot.retry_at = payload["retry_at"]
+        update_fields.append("retry_at")
 
-        if tags is not None:
-            snapshot.save_tags(
-                normalize_tag_list(tags),
-                created_by=request.user if isinstance(request.user, User) else None,
-            )
+    if tags is not None:
+        snapshot.save_tags(normalize_tag_list(tags))
 
-        if payload.get("status") == Snapshot.StatusChoices.SEALED:
-            snapshot.cancel()
-        else:
-            snapshot.save(update_fields=update_fields)
+    if payload.get("status") == Snapshot.StatusChoices.SEALED:
+        snapshot.cancel()
+    else:
+        snapshot.save(update_fields=update_fields)
     setattr(request, "with_archiveresults", False)
     return snapshot
 
 
 @router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
 def delete_snapshot(request: HttpRequest, snapshot_id: str):
-    """Queue deletion; the normal server runner stops writers and removes files.
-
-    The snapshot remains readable with status=deleting until cleanup succeeds.
-    queued_count reports acceptance; deleted_count is zero in this response.
-    """
     snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
     snapshot_id_str = str(snapshot.id)
-    crawl_id_str = str(snapshot.crawl_id)
-    # Use the same durable queue as the admin delete action. Stopping writers
-    # and removing archive files belongs to the runner, never the HTTP request.
-    queued_count = Snapshot.objects.filter(pk=snapshot.pk).request_delete()
+    crawl_id_str = str(snapshot.crawl.pk)
+    deleted_count, _ = snapshot.delete()
     return {
         "success": True,
         "snapshot_id": snapshot_id_str,
         "crawl_id": crawl_id_str,
-        "deleted_count": 0,
-        "queued_count": queued_count,
+        "deleted_count": deleted_count,
     }
 
 
@@ -1121,7 +1057,7 @@ class TagSchema(Schema):
     def resolve_created_by_username(obj):
         user_model = get_user_model()
         user = user_model.objects.get(id=obj.created_by_id)
-        username = user.username
+        username = getattr(user, "username", None)
         return username if isinstance(username, str) else str(user)
 
     @staticmethod
@@ -1130,7 +1066,7 @@ class TagSchema(Schema):
 
     @staticmethod
     def resolve_snapshots(obj, context):
-        if bool(context["request"].__dict__.get("with_snapshots", False)):
+        if bool(getattr(context["request"], "with_snapshots", False)):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
@@ -1273,7 +1209,7 @@ def _get_snapshot_for_tag_edit(snapshot_ref: str) -> Snapshot:
     is_full_uuid = len(snapshot_ref.replace("-", "")) == 32 and all(char in "0123456789abcdef-" for char in snapshot_ref)
     if is_full_uuid:
         try:
-            return snapshot_qs.get(pk=snapshot_ref.replace("-", ""))
+            return snapshot_qs.get(pk=snapshot_ref)
         except (Snapshot.DoesNotExist, ValueError):
             pass
 
@@ -1316,7 +1252,6 @@ def search_tags(
         "tags": build_tag_cards(
             query=q,
             request=request,
-            preview_limit=0,
             sort=normalized_sort,
             created_by=normalized_created_by,
             year=normalized_year,
@@ -1334,7 +1269,16 @@ def _public_tag_listing_enabled() -> bool:
 
 
 def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    if authenticated_user_from_request(request):
+    user = getattr(request, "user", None)
+    if getattr(user, "is_authenticated", False):
+        return True
+
+    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
+    auth_header = request.headers.get("Authorization", "")
+    if not token and auth_header.lower().startswith("bearer "):
+        token = auth_header.split(None, 1)[1].strip()
+
+    if token and auth_using_token(token=token, request=request):
         return True
 
     return _public_tag_listing_enabled()
@@ -1346,8 +1290,8 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     if not _request_has_tag_autocomplete_access(request):
         raise HttpError(401, "Authentication required")
 
-    public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
-    queryset = get_matching_tags(q)
+    public_only = not getattr(request.user, "is_authenticated", False) and not getattr(request, "_api_token", None)
+    queryset = get_matching_tags(q, with_snapshot_counts=False)
     public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
     if public_only:
         queryset = queryset.filter(snapshot_set__id__in=public_snapshots.values("id")).distinct()
@@ -1355,7 +1299,7 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
 
     return {
-        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
+        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": getattr(tag, "num_snapshots", 0)} for tag in tags],
     }
 
 
@@ -1455,7 +1399,8 @@ def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
     else:
         raise HttpError(400, "Either tag_name or tag_id is required")
 
-    snapshot.add_tag_ids([tag.pk])
+    # Add the tag to the snapshot
+    snapshot.tags.add(tag.pk)
 
     return {
         "success": True,
@@ -1483,7 +1428,8 @@ def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSche
     else:
         raise HttpError(400, "Either tag_name or tag_id is required")
 
-    snapshot.remove_tag_ids([tag.pk])
+    # Remove the tag from the snapshot
+    snapshot.tags.remove(tag.pk)
 
     return {
         "success": True,

archivebox/api/v1_crawls.py

@@ -14,17 +14,10 @@ from ninja import Router, Schema
 from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot
-from archivebox.core.permissions import (
-    PERMISSIONS_PUBLIC,
-    PERMISSIONS_UNLISTED,
-    is_admin_user,
-    normalize_permissions,
-)
 from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
-from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url
 
-from .auth import API_AUTH_METHODS, authenticated_user_from_request
+from .auth import API_AUTH_METHODS, auth_using_token
 
 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
 
@@ -48,6 +41,8 @@ class CrawlSchema(Schema):
     tags_str: str
     config: dict
 
+    # snapshots: List[SnapshotSchema]
+
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
@@ -56,20 +51,12 @@ class CrawlSchema(Schema):
     def resolve_created_by_username(obj):
         user_model = get_user_model()
         user = user_model.objects.get(id=obj.created_by_id)
-        username = user.username
+        username = getattr(user, "username", None)
         return username if isinstance(username, str) else str(user)
 
-    @staticmethod
-    def resolve_config(obj):
-        # Redact credential values so REST responses can never leak the raw
-        # token/secret/api-key that the operator stored in Crawl.config.
-        from archivebox.config.common import redact_sensitive_config
-
-        return redact_sensitive_config(obj.config)
-
     @staticmethod
     def resolve_snapshots(obj, context):
-        if bool(context["request"].__dict__.get("with_snapshots", False)):
+        if bool(getattr(context["request"], "with_snapshots", False)):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
@@ -112,10 +99,7 @@ def get_crawls(request: HttpRequest):
 
 @router.post("/crawls", response=CrawlSchema, url_name="create_crawl")
 def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
-    try:
-        urls = [validate_url(url) for url in data.urls if url and url.strip()]
-    except ValueError as err:
-        raise HttpError(400, str(err)) from err
+    urls = [url.strip() for url in data.urls if url and url.strip()]
     if not urls:
         raise HttpError(400, "At least one URL is required")
     if data.max_depth not in (0, 1, 2, 3, 4):
@@ -123,8 +107,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
 
     tags = normalize_tag_list(data.tags, data.tags_str)
     config = dict(data.config or {})
-    config.setdefault("PARSER", "url_list")
-    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
+    config.setdefault("PERMISSIONS", str(get_config(user=request.user).PERMISSIONS))
     crawl = Crawl.objects.create(
         urls="\n".join(urls),
         max_depth=data.max_depth,
@@ -139,16 +122,12 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
     return crawl
 
 
-def get_crawl_by_ref(crawl_id: str):
-    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
-
-
 @router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
 def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
     """Get a specific Crawl by id."""
     setattr(request, "with_snapshots", with_snapshots)
     setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = get_crawl_by_ref(crawl_id)
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
 
     if crawl and as_rss:
         query = request.GET.copy()
@@ -160,36 +139,26 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
 
 
 def crawl_file(request: HttpRequest, crawl_id: str, path: str):
-    # Try to resolve the crawl first; if it doesn't exist, return 404.
-    try:
-        crawl = get_crawl_by_ref(crawl_id)
-    except Crawl.DoesNotExist:
-        raise HttpError(404, "Crawl not found")
-
-    user = authenticated_user_from_request(request)
-
-    # Gate access using the same model as SnapshotView/can_view_snapshot:
-    # admins always pass; owners can see their own crawls; otherwise the crawl
-    # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
-    if not is_admin_user(request):
-        permissions = normalize_permissions(crawl.permissions)
-        is_owner = bool(user and crawl.created_by_id == user.id)
-        if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
-            raise HttpError(404, "Crawl not found")
+    user = getattr(request, "user", None)
+    is_superuser = bool(
+        getattr(user, "is_authenticated", False) and getattr(user, "is_active", False) and getattr(user, "is_superuser", False),
+    )
+    if not is_superuser:
+        token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
+        auth_header = request.headers.get("Authorization", "")
+        if not token and auth_header.lower().startswith("bearer "):
+            token = auth_header.split(None, 1)[1].strip()
+        token_user = auth_using_token(token=token, request=request) if token else None
+        is_superuser = bool(token_user and token_user.is_active and token_user.is_superuser)
+    if not is_superuser:
+        raise HttpError(403, "Permission denied")
 
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
     crawl_root = Path(crawl.output_dir).resolve()
     file_path = (crawl_root / path).resolve()
     if not file_path.is_file() or crawl_root not in file_path.parents:
         raise HttpError(404, "Crawl file not found")
 
-    if path == "chrome_screencast/latest.jpg" and request.GET.get("after"):
-        try:
-            started_at = float(request.GET["after"])
-        except ValueError:
-            raise HttpError(400, "Invalid screencast start time")
-        if file_path.stat().st_mtime < started_at:
-            raise HttpError(404, "No frame for this capture yet")
-
     response = FileResponse(file_path.open("rb"))
     response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
     response["Pragma"] = "no-cache"
@@ -216,7 +185,7 @@ def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfol
 @router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
 def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
     """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = get_crawl_by_ref(crawl_id)
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
 
@@ -258,18 +227,10 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
 
 @router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
 def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = get_crawl_by_ref(crawl_id)
+    crawl = Crawl.objects.get(id__icontains=crawl_id)
     crawl_id_str = str(crawl.id)
-    crawl.cancel()
-
-    from archivebox.crawls.locks import crawl_lifecycle_lock
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        crawl = get_crawl_by_ref(crawl_id_str)
-        snapshot_count = crawl.snapshot_set.count()
-        deleted_count, _ = crawl.delete()
+    snapshot_count = crawl.snapshot_set.count()
+    deleted_count, _ = crawl.delete()
     return {
         "success": True,
         "crawl_id": crawl_id_str,

archivebox/api/v1_personas.py

@@ -2,18 +2,14 @@ __package__ = "archivebox.api"
 
 import json
 from datetime import datetime
-from typing import Any, Literal
+from typing import Any
 from uuid import UUID
 
-from django.core.exceptions import ValidationError
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
-from ninja.errors import HttpError
-from ninja.pagination import paginate
 from pydantic import Field
 
-from archivebox.api.v1_core import CustomPagination
 from archivebox.personas.importers import validate_persona_name
 from archivebox.personas.models import Persona
 
@@ -25,7 +21,6 @@ class PersonaBrowserSettingsSchema(Schema):
     user_agent: str = ""
     viewport_size: str = ""
     viewport_device_scale_factor: float | None = None
-    color_scheme: Literal["", "light", "dark"] = ""
     language: str = ""
     timezone: str = ""
     geolocation: dict[str, Any] | None = None
@@ -56,14 +51,6 @@ class PersonaSchema(Schema):
     def resolve_created_by_username(obj) -> str:
         return obj.created_by.username
 
-    @staticmethod
-    def resolve_config(obj):
-        # Redact credential values so REST responses don't leak the raw
-        # token/secret/api-key the operator stored in Persona.config.
-        from archivebox.config.common import redact_sensitive_config
-
-        return redact_sensitive_config(obj.config)
-
 
 class PersonaSyncResponseSchema(Schema):
     success: bool
@@ -97,8 +84,6 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
         )
     if settings.viewport_device_scale_factor is not None:
         config["BROWSER_DEVICE_SCALE_FACTOR"] = settings.viewport_device_scale_factor
-    if "color_scheme" in settings.model_fields_set:
-        config["BROWSER_COLOR_SCHEME"] = settings.color_scheme
     if settings.language:
         config["BROWSER_LANGUAGE"] = settings.language
     if settings.timezone:
@@ -110,16 +95,16 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
 
 
 def find_persona(extension_persona_id: str, name: str) -> Persona | None:
-    named = Persona.find_named(name)
     return (
-        Persona.objects.filter(Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(pk=named.pk if named else None))
+        Persona.objects.filter(
+            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
+        )
         .order_by("created_at")
         .first()
     )
 
 
 @router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
-@paginate(CustomPagination)
 def get_personas(request: HttpRequest):
     """List personas available on this ArchiveBox server."""
     return Persona.objects.all().order_by("name")
@@ -139,15 +124,11 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     if not is_valid:
         raise ValueError(error_message)
 
-    try:
-        persona = find_persona(payload.extension_persona_id, name)
-        (persona or Persona(name=name)).validate_name(persona.name if persona else name)
-    except ValidationError as err:
-        raise HttpError(409, "; ".join(err.messages)) from err
+    persona = find_persona(payload.extension_persona_id, name)
     created = persona is None
     if persona is None:
         persona = Persona(name=name)
-        if request.user.is_authenticated:
+        if getattr(request.user, "is_authenticated", False):
             persona.created_by = request.user
 
     persona.config = {
@@ -158,12 +139,12 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     persona.ensure_dirs()
 
     cookies_written = False
-    if "cookies_txt" in payload.model_fields_set:
+    if payload.cookies_txt.strip():
         (persona.path / "cookies.txt").write_text(payload.cookies_txt)
         cookies_written = True
 
     auth_written = False
-    if "auth_json" in payload.model_fields_set:
+    if payload.auth_json:
         (persona.path / "auth.json").write_text(json.dumps(payload.auth_json, indent=2, sort_keys=True) + "\n")
         auth_written = True
 

archivebox/api/webhooks.py

@@ -5,40 +5,12 @@ from collections.abc import Callable
 import logging
 
 from django.db import transaction
-from django.db.models.signals import pre_delete
 from signal_webhooks.handlers import sync_task_handler
 
 
 logger = logging.getLogger(__name__)
 
 
-def load_deferred_fields_for_delete_webhook(sender, instance, using, **kwargs) -> None:
-    """Materialize deferred fields while DELETE webhook rows still exist.
-
-    Admin lists defer large fields, but signal_webhooks serializes all concrete
-    fields in post_delete. Loading one then would query an already-deleted row
-    and drop the event. Hydrate only deferred fields on webhook-enabled models
-    here, preserving the normal list-query optimization.
-    """
-    from signal_webhooks.handlers import find_hook_handler
-    from signal_webhooks.utils import reference_for_model
-
-    ref = reference_for_model(type(instance))
-    deferred_fields = instance.get_deferred_fields()
-    if not deferred_fields or find_hook_handler(ref, "DELETE") is None:
-        return
-
-    instance.refresh_from_db(using=using, fields=deferred_fields)
-
-
-def register_delete_webhook_field_loader() -> None:
-    pre_delete.connect(
-        load_deferred_fields_for_delete_webhook,
-        dispatch_uid="archivebox.load_deferred_fields_for_delete_webhook",
-        weak=False,
-    )
-
-
 def warning_error_handler(hook: Any, error: Exception | None) -> None:
     if error is not None:
         logger.warning("Outbound webhook %r failed: %s", hook.name, error)

archivebox/base_models/admin.py

@@ -3,50 +3,18 @@
 __package__ = "archivebox.base_models"
 
 import json
-import uuid
 from collections.abc import Mapping
-from typing import ClassVar, NotRequired, TypedDict, cast
+from typing import NotRequired, TypedDict
 
 from django import forms
 from django.contrib import admin
-from django.db import DatabaseError, models
+from django.db import models
 from django.forms.renderers import BaseRenderer
 from django.http import HttpRequest, QueryDict
-from django.urls import path, register_converter
 from django.utils.safestring import SafeString, mark_safe
 from django_object_actions import DjangoObjectActions
 
 
-class HexUUIDConverter:
-    """URL path converter that canonicalizes UUIDs to their 32-char hex form.
-
-    Accepts both the hyphenated (``aaaaaaaa-bbbb-...``) and bare-hex
-    (``aaaaaaaabbbb...``) UUID strings on the way in (Django's UUIDField
-    parses either), but ``to_url`` always emits the bare-hex form. This is
-    what makes ``reverse("admin:app_model_change", args=[obj.pk])`` produce
-    ``/admin/app/model/06a1a8facb0d.../change/`` instead of the default
-    hyphenated rendering — admin links throughout the app reverse through
-    this converter once ``BaseModelAdmin.get_urls`` swaps in
-    ``<hexuuid:object_id>`` below.
-    """
-
-    regex = r"[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
-
-    def to_python(self, value: str) -> str:
-        # Strip hyphens but stay as a string — Django admin treats the
-        # captured object_id as a string and calls ``model._meta.pk.to_python``
-        # itself, so we don't want to short-circuit that.
-        return value.replace("-", "")
-
-    def to_url(self, value) -> str:
-        if isinstance(value, uuid.UUID):
-            return value.hex
-        return str(value).replace("-", "")
-
-
-register_converter(HexUUIDConverter, "hexuuid")
-
-
 class ConfigOption(TypedDict):
     plugin: str
     type: str | list[str]
@@ -68,33 +36,49 @@ class KeyValueWidget(forms.Widget):
     template_name = ""  # We render manually
 
     class Media:
-        css: ClassVar[dict[str, list[str]]] = {
+        css = {
             "all": [],
         }
-        js: ClassVar[list[str]] = []
+        js = []
 
     def _get_config_options(self) -> dict[str, ConfigOption]:
         """Get available config options from plugins."""
         try:
-            from archivebox.config.common import config_field_metadata
+            from archivebox.config.common import ArchiveBoxConfig
+            from archivebox.hooks import discover_plugin_configs
 
             options: dict[str, ConfigOption] = {}
-            for key, metadata in config_field_metadata().items():
-                option_type = metadata.get("type", "string")
-                option: ConfigOption = {
-                    "plugin": str(metadata.get("plugin", "archivebox")),
-                    "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
-                    "default": metadata.get("default", ""),
-                    "description": str(metadata.get("description", "")),
+            skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "CRAWL_OUTPUT_DIR", "SNAP_DIR"}
+            for key, field in ArchiveBoxConfig.model_fields.items():
+                if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
+                    continue
+                default = field.default
+                try:
+                    json.dumps(default)
+                except TypeError:
+                    default = str(default)
+                options[key] = {
+                    "plugin": "archivebox",
+                    "type": str(field.annotation),
+                    "default": default,
+                    "description": field.description or "",
                 }
-                schema = metadata.get("schema")
-                if isinstance(schema, Mapping):
+
+            plugin_configs = discover_plugin_configs()
+            for plugin_name, schema in plugin_configs.items():
+                for key, prop in schema.get("properties", {}).items():
+                    option: ConfigOption = {
+                        "plugin": plugin_name,
+                        "type": prop.get("type", "string"),
+                        "default": prop.get("default", ""),
+                        "description": prop.get("description", ""),
+                    }
                     for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in schema:
-                            option[schema_key] = schema[schema_key]
-                options[key] = option
+                        if schema_key in prop:
+                            option[schema_key] = prop[schema_key]
+                    options[key] = option
             return options
-        except (ImportError, KeyError, TypeError, ValueError):
+        except Exception:
             return {}
 
     def _parse_value(self, value: object) -> dict[str, object]:
@@ -243,6 +227,9 @@ class KeyValueWidget(forms.Widget):
                             return 'Example: ["value"]';
                         }}
                         if (types.includes('object')) {{
+                            if (key === 'SAVE_ALLOWLIST' || key === 'SAVE_DENYLIST') {{
+                                return 'Example: {{"^https://example\\\\.com": ["wget"]}}';
+                            }}
                             return 'Example: {{"key": "value"}}';
                         }}
                         return '';
@@ -251,6 +238,8 @@ class KeyValueWidget(forms.Widget):
                     function isRegexConfigKey_{widget_id}(key) {{
                         return key === 'URL_ALLOWLIST' ||
                             key === 'URL_DENYLIST' ||
+                            key === 'SAVE_ALLOWLIST' ||
+                            key === 'SAVE_DENYLIST' ||
                             key.endsWith('_PATTERN') ||
                             key.includes('REGEX');
                     }}
@@ -644,99 +633,17 @@ class KeyValueWidget(forms.Widget):
 
                     window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
 
-                    function configRowForKey_{widget_id}(key) {{
-                        var container = document.getElementById('{widget_id}_rows');
-                        if (!container) {{
-                            return null;
-                        }}
-                        var match = null;
-                        container.querySelectorAll('.key-value-row').forEach(function(row) {{
-                            if (match) {{ return; }}
-                            var keyInput = row.querySelector('.kv-key');
-                            if (keyInput && keyInput.value.trim() === key) {{
-                                match = row;
-                            }}
-                        }});
-                        if (!match) {{
-                            window.addKeyValueRow_{widget_id}();
-                            var rows = container.querySelectorAll('.key-value-row');
-                            match = rows[rows.length - 1];
-                            var keyInput = match.querySelector('.kv-key');
-                            if (keyInput) {{
-                                keyInput.value = key;
-                                keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
-                            }}
-                        }}
-                        return match;
-                    }}
-
-                    function prefillConfigFromQuery_{widget_id}() {{
-                        var params = new URLSearchParams(window.location.search);
-                        var consumedConfigKeys = [];
-                        params.forEach(function(value, key) {{
-                            if (!/^[A-Z][A-Z0-9_]*$/.test(key) || !configMeta_{widget_id}[key]) {{
-                                return;
-                            }}
-                            consumedConfigKeys.push(key);
-                            var match = configRowForKey_{widget_id}(key);
-                            var valueInput = match && match.querySelector('.kv-value');
-                            if (valueInput) {{
-                                valueInput.value = value;
-                                valueInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
-                            }}
-                        }});
-                        updateHiddenField_{widget_id}();
-                        if (consumedConfigKeys.length) {{
-                            var cleanUrl = new URL(window.location.href);
-                            consumedConfigKeys.forEach(function(key) {{ cleanUrl.searchParams.delete(key); }});
-                            window.history.replaceState(null, '', cleanUrl.pathname + cleanUrl.search + cleanUrl.hash);
-                        }}
-                    }}
-
-                    function focusConfigKeyFromHash_{widget_id}() {{
-                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
-                        // to (or creates) the matching row in this editor. Used by
-                        // the setup wizard and live-config detail pages.
-                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
-                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
-                            return;
-                        }}
-                        var match = configRowForKey_{widget_id}(hash);
-                        if (!match) {{
-                            return;
-                        }}
-                        match.scrollIntoView({{ behavior: 'smooth', block: 'center' }});
-                        var prevOutline = match.style.outline;
-                        match.style.outline = '2px solid #f59e0b';
-                        match.style.outlineOffset = '2px';
-                        match.style.transition = 'outline 1.2s ease-out';
-                        setTimeout(function() {{
-                            match.style.outline = prevOutline || 'none';
-                        }}, 1400);
-                        var valueInput = match.querySelector('.kv-value');
-                        if (valueInput) {{
-                            valueInput.focus();
-                            try {{ valueInput.setSelectionRange(valueInput.value.length, valueInput.value.length); }} catch (e) {{}}
-                        }}
-                    }}
-
                     // Initialize on load
                     document.addEventListener('DOMContentLoaded', function() {{
                         initializeRows_{widget_id}();
-                        prefillConfigFromQuery_{widget_id}();
                         updateHiddenField_{widget_id}();
-                        focusConfigKeyFromHash_{widget_id}();
                     }});
                     // Also run immediately in case DOM is already ready
                     if (document.readyState !== 'loading') {{
                         initializeRows_{widget_id}();
-                        prefillConfigFromQuery_{widget_id}();
                         updateHiddenField_{widget_id}();
-                        focusConfigKeyFromHash_{widget_id}();
                     }}
 
-                    window.addEventListener('hashchange', focusConfigKeyFromHash_{widget_id});
-
                     // Update on any input change
                     var rowsEl_{widget_id} = document.getElementById('{widget_id}_rows');
 
@@ -763,34 +670,12 @@ class KeyValueWidget(forms.Widget):
         return mark_safe(html)
 
     def _render_row(self, widget_id: str, key: str, value: str) -> str:
-        from archivebox.config.common import is_sensitive_config_key
-
-        # Sensitive keys (``*TOKEN*``, ``*SECRET*``, ``*API_KEY*``, ``*APIKEY*``) are
-        # rendered write-only: the input is a password field with a placeholder
-        # showing the value is set, but the raw value is NEVER sent to the browser.
-        # When the user submits the form with the field left blank, the
-        # ``ConfigEditorMixin.save_model`` hook re-merges the previously-saved
-        # value so leaving it untouched is a no-op rather than a destructive clear.
-        is_sensitive = is_sensitive_config_key(key)
-        has_value = bool(value)
-        if is_sensitive:
-            input_type = "password"
-            rendered_value = ""
-            placeholder = (
-                "•••••• (saved — enter new value to replace, clear by deleting row)" if has_value else "value (will be saved write-only)"
-            )
-            extra_attrs = ' autocomplete="off" data-sensitive="1"' + (' data-had-value="1"' if has_value else "")
-        else:
-            input_type = "text"
-            rendered_value = self._escape(value)
-            placeholder = "value"
-            extra_attrs = ""
         return f'''
             <div class="key-value-row" style="margin-bottom: 6px;">
                 <div class="kv-inputs" style="display: flex; gap: 8px; align-items: center;">
                     <input type="text" class="kv-key" value="{self._escape(key)}" placeholder="KEY" list="{widget_id}_keys"
                            style="flex: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
-                    <input type="{input_type}" class="kv-value" value="{rendered_value}" placeholder="{self._escape(placeholder)}"{extra_attrs}
+                    <input type="text" class="kv-value" value="{self._escape(value)}" placeholder="value"
                            style="flex: 2; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;">
                     <datalist class="kv-value-options"></datalist>
                     <button type="button" onclick="removeKeyValueRow_{widget_id}(this)"
@@ -834,56 +719,11 @@ class ConfigEditorMixin(admin.ModelAdmin):
             kwargs["widget"] = KeyValueWidget()
         return super().formfield_for_dbfield(db_field, request, **kwargs)
 
-    def save_model(self, request: HttpRequest, obj, form, change):
-        """Preserve write-only redacted credentials on save.
-
-        The KeyValueWidget renders sensitive keys (``*TOKEN*``, ``*SECRET*``,
-        ``*API_KEY*``, ``*APIKEY*``) with an empty value + password input —
-        the real value never leaves the server. On submit, an empty value
-        for a sensitive key that was previously set means "leave untouched",
-        not "clear it." We honor that here by re-merging the stored value
-        before the row is written. Explicitly removing the row in the UI
-        still clears it (the key is gone from the submitted JSON, so there's
-        nothing to merge over).
-        """
-        from archivebox.config.common import is_sensitive_config_key
-
-        if change and obj.pk and obj.config is not None:
-            try:
-                stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
-            except (AttributeError, DatabaseError, TypeError, ValueError):
-                stored = {}
-            if isinstance(stored, dict):
-                new_config = dict(obj.config or {})
-                for key, new_value in list(new_config.items()):
-                    if not is_sensitive_config_key(key):
-                        continue
-                    if new_value not in (None, "") and new_value != "********":
-                        continue
-                    if key in stored:
-                        new_config[key] = stored[key]
-                obj.config = new_config
-        super().save_model(request, obj, form, change)
-
 
 class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
     list_display = ("id", "created_at", "created_by")
     readonly_fields = ("id", "created_at", "modified_at")
     show_search_mode_selector = False
-    change_form_template = "admin/archivebox_change_form.html"
-
-    def get_admin_toolbar_actions(self, request, obj):
-        """Return extra action button dicts for the shared change-form toolbar.
-
-        See ``templates/admin/includes/archivebox_toolbar.html`` for the
-        accepted keys. Default: no extras (toolbar renders Save/History/Delete
-        plus any ``django-object-actions`` change_actions).
-        """
-        return []
-
-    def render_change_form(self, request, context, add=False, change=False, form_url="", obj=None):
-        context.setdefault("archivebox_admin_actions", self.get_admin_toolbar_actions(request, obj))
-        return super().render_change_form(request, context, add=add, change=change, form_url=form_url, obj=obj)
 
     def get_default_search_mode(self) -> str:
         # The shared changelist template always asks every admin for a default
@@ -901,30 +741,3 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
         if "created_by" in form.base_fields:
             form.base_fields["created_by"].initial = request.user
         return form
-
-    def get_urls(self):
-        """Swap the per-object admin URLs from ``<path:object_id>`` to
-        ``<hexuuid:object_id>`` so canonical change/delete/history URLs use the
-        32-char hex form. The hyphenated form still resolves because the
-        converter's regex accepts both — Django reverses through ``to_url``
-        which always emits hex, so links in templates / changelists / inline
-        formsets all canonicalize automatically.
-
-        Non-UUID PKs (an ``IntegerField`` PK on some legacy table, for example)
-        won't match the converter's regex and fall back to the default
-        ``<path:object_id>`` patterns we still include after our swap.
-        """
-        info = self.opts.app_label, self.opts.model_name
-        object_routes = [
-            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="{}_{}_history".format(*info)),
-            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="{}_{}_delete".format(*info)),
-            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="{}_{}_change".format(*info)),
-        ]
-        # Append after super().get_urls() so our patterns are the
-        # *last-registered* ones with the canonical admin URL names — Django's
-        # reverse() picks the later registration when names collide, which is
-        # how we make ``reverse("admin:app_model_change", args=[obj.pk])``
-        # emit the hex form. The original ``<path:object_id>`` routes stay in
-        # place as a fallback for non-UUID PKs and for resolving inbound
-        # hyphenated URLs (the ``hexuuid`` regex accepts both forms anyway).
-        return super().get_urls() + object_routes

archivebox/base_models/apps.py

@@ -0,0 +1,7 @@
+# from django.apps import AppConfig
+
+
+# class BaseModelsConfig(AppConfig):
+#     default_auto_field = 'django.db.models.BigAutoField'
+
+#     name = 'base_models'

archivebox/base_models/models.py

@@ -2,14 +2,11 @@
 
 __package__ = "archivebox.base_models"
 
-import json
 import shutil
-from typing import Any
 
-from archivebox.uuid_compat import CompactUUIDField, uuid7
+from archivebox.uuid_compat import uuid7
 from pathlib import Path
 
-from django.core.exceptions import FieldDoesNotExist
 from django.db import models
 from django.db.models import F
 from django.db import transaction
@@ -21,131 +18,6 @@ from django.conf import settings
 
 from django_stubs_ext.db.models import TypedModelMeta
 
-from archivebox.config import CONSTANTS
-
-
-class PagedQuerySet(models.QuerySet):
-    def paged_iterator(self, chunk_size: int = 500):
-        """
-        Iterate rows using bounded keyset pages instead of one streaming cursor.
-
-        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
-        cursor open until the full queryset is exhausted. That is fine for
-        read-only exports, but update/migration code does filesystem work and
-        writes while iterating; a long-lived read cursor there can stretch lock
-        waits across thousands of rows. This respects the queryset's existing
-        filters, order_by(), select_related(), and prefetch_related() state; if
-        no ordering is defined, it falls back to primary-key order.
-        """
-        pk_field = self.model._meta.pk.name
-        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
-
-        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        ordering = []
-        for term in raw_ordering:
-            descending = term.startswith("-")
-            field_name = term[1:] if descending else term
-            if field_name == "pk":
-                field_name = pk_field
-            ordering.append(f"-{field_name}" if descending else field_name)
-
-        ordered_field_names = [term.removeprefix("-") for term in ordering]
-        try:
-            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
-                offset = 0
-                while True:
-                    batch = list(self[offset : offset + chunk_size])
-                    if not batch:
-                        break
-                    yield from batch
-                    offset += chunk_size
-                return
-        except (AttributeError, FieldDoesNotExist):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
-        if not any(field_name in unique_field_names for field_name in ordered_field_names):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        last_values = None
-        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
-        while True:
-            batch_qs = self.order_by(*ordering)
-            if last_values is not None:
-                page_filter = models.Q()
-                for idx, term in enumerate(ordering):
-                    descending = term.startswith("-")
-                    field_name = term[1:] if descending else term
-                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
-                    comparison = "lt" if descending else "gt"
-                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
-                # Anchor the OR tie-breaks with an explicit leading range so
-                # SQLite/PG can seek the existing order index instead of
-                # rescanning its prefix for every keyset page.
-                first_comparison = "lte" if ordering[0].startswith("-") else "gte"
-                batch_qs = batch_qs.filter(
-                    page_filter,
-                    **{f"{ordered_field_names[0]}__{first_comparison}": last_values[0]},
-                )
-
-            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
-            if not batch_rows:
-                break
-
-            pk_idx = value_field_names.index(pk_field)
-            row_ids = [row[pk_idx] for row in batch_rows]
-            rows_by_id = {obj.pk: obj for obj in self.filter(pk__in=row_ids).order_by()}
-
-            for row in batch_rows:
-                row_id = row[pk_idx]
-                obj = rows_by_id.get(row_id)
-                if obj is not None:
-                    yield obj
-
-            last_values = batch_rows[-1][: len(ordered_field_names)]
-
-
-def normalize_config_json_values(config: Any) -> Any:
-    if not isinstance(config, dict):
-        return config
-
-    normalized = dict(config)
-    for key, value in list(normalized.items()):
-        if not isinstance(value, str) or len(value) < 2:
-            continue
-        if value[:1] != '"' or value[-1:] != '"':
-            continue
-        try:
-            decoded = json.loads(value)
-        except ValueError:
-            continue
-        if isinstance(decoded, str):
-            normalized[key] = decoded
-    return normalized
-
 
 def get_or_create_system_user_pk(username="system"):
     User = get_user_model()
@@ -172,7 +44,7 @@ class AutoDateTimeField(models.DateTimeField):
 
 
 class ModelWithUUID(models.Model):
-    id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
+    id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     modified_at = models.DateTimeField(auto_now=True)
     created_by = models.ForeignKey(
@@ -220,10 +92,6 @@ class ModelWithHealthStats(models.Model):
     class Meta(TypedModelMeta):
         abstract = True
 
-    @property
-    def admin_change_url(self) -> str:
-        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
-
     @property
     def health(self) -> int:
         total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
@@ -248,15 +116,6 @@ class ModelWithConfig(models.Model):
     class Meta(TypedModelMeta):
         abstract = True
 
-    def save(self, *args, **kwargs):
-        normalized_config = normalize_config_json_values(self.config)
-        if normalized_config != self.config:
-            self.config = normalized_config
-            update_fields = kwargs.get("update_fields")
-            if update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
-        super().save(*args, **kwargs)
-
 
 class ModelWithDeleteAfter(models.Model):
     delete_after_final_statuses: tuple[str, ...] = ()
@@ -276,7 +135,7 @@ class ModelWithDeleteAfter(models.Model):
     def get_delete_after_config_value(self):
         from archivebox.config.common import get_config
 
-        return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
+        return get_config(include_machine=False).DELETE_AFTER
 
     def set_delete_at_from_config(self, config_value=None) -> bool:
         if self.delete_at is not None:
@@ -296,21 +155,9 @@ class ModelWithDeleteAfter(models.Model):
         return cls.objects.none()
 
     @classmethod
-    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True, backfill_cursors: dict | None = None) -> int:
+    def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
         if backfill_missing:
-            # Bound the scanned rows BEFORE evaluating JSON policy/parent
-            # joins. LIMIT on the policy query only bounded its output: finding
-            # no matches could scan the entire history with a SQLite read lock.
-            # The runner keeps a keyset cursor between idle passes, so disabled
-            # policies cannot starve older rows. Restarting safely repeats work.
-            page = cls.objects.order_by("-pk")
-            before = backfill_cursors.get(cls) if backfill_cursors is not None else None
-            if before is not None:
-                page = page.filter(pk__lt=before)
-            page_ids = list(page.values_list("pk", flat=True)[:batch_size])
-            if backfill_cursors is not None:
-                backfill_cursors[cls] = page_ids[-1] if len(page_ids) == batch_size else None
-            missing_delete_at = list(cls.missing_delete_at_candidates().filter(pk__in=page_ids).order_by())
+            missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
             for obj in missing_delete_at:
                 if obj.set_delete_at_from_config():
                     cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
@@ -348,9 +195,12 @@ class ModelWithOutputDir(ModelWithUUID):
 
     _delete_signal_registered = False
 
-    # Saving metadata must not resolve or create output paths. The runner and
-    # explicit artifact writers create their own directories when writing files;
-    # on_commit would still make HTTP requests wait on remote archive storage.
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        output_dir = Path(self.output_dir)
+        # Avoid holding SQLite write transactions open across slow filesystem work.
+        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
+        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
 
     @property
     def output_dir_parent(self) -> str:
@@ -373,7 +223,9 @@ class ModelWithOutputDir(ModelWithUUID):
 
     @classmethod
     def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
-        data_dir = CONSTANTS.DATA_DIR.resolve()
+        from archivebox.config.common import get_config
+
+        data_dir = get_config().DATA_DIR.resolve()
         safe_paths = []
         for raw_path in paths:
             path = Path(raw_path)
@@ -396,31 +248,18 @@ class ModelWithOutputDir(ModelWithUUID):
             if path.is_symlink() or path.is_file():
                 path.unlink(missing_ok=True)
             elif path.is_dir():
-                shutil.rmtree(path)
-            # lstat also catches dangling symlinks and propagates mount and
-            # permission errors rather than mistaking them for absence.
-            try:
-                path.lstat()
-            except FileNotFoundError:
-                continue
-            raise OSError(f"Output path still exists after deletion: {path}")
-
-    def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
-        """Capture output paths before DB deletion and remove them after commit."""
-        paths = self.validate_output_paths_for_delete(self.output_paths_for_delete())
-        transaction.on_commit(lambda: self.delete_output_paths(paths), using=using)
+                shutil.rmtree(path, ignore_errors=True)
 
     @classmethod
     def register_delete_signal(cls) -> None:
         if cls._delete_signal_registered:
             return
 
-        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
-            if getattr(origin, "_output_files_deleted", False):
-                return
+        def schedule_output_dir_cleanup(sender, instance, **kwargs):
             if not isinstance(instance, ModelWithOutputDir):
                 return
-            instance.schedule_delete_cleanup(using=using)
+            paths = instance.validate_output_paths_for_delete(instance.output_paths_for_delete())
+            transaction.on_commit(lambda paths=paths: instance.delete_output_paths(paths))
 
         pre_delete.connect(
             schedule_output_dir_cleanup,

archivebox/cli/__init__.py

@@ -2,7 +2,6 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox"
 import os
 import sys
-from contextlib import redirect_stdout
 from importlib import import_module
 
 import rich_click as click
@@ -17,15 +16,6 @@ if "--debug" in sys.argv:
     os.environ["DEBUG"] = "True"
     sys.argv.remove("--debug")
 
-# Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
-# `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
-# the subcommand executes. `--quick-init` is the old server spelling. Strip either from
-# argv here so each subcommand's own click parser never sees it. Ignored for `help` and
-# `init` themselves.
-if "--init" in sys.argv or "--quick-init" in sys.argv:
-    sys.argv = [arg for arg in sys.argv if arg not in ("--init", "--quick-init")]
-    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
-
 
 class ArchiveBoxGroup(click.Group):
     """lazy loading click group for archivebox commands"""
@@ -34,7 +24,6 @@ class ArchiveBoxGroup(click.Group):
         "help": "archivebox.cli.archivebox_help.main",
         "version": "archivebox.cli.archivebox_version.main",
         "mcp": "archivebox.cli.archivebox_mcp.main",
-        "oneshot": "archivebox.cli.archivebox_oneshot.main",
     }
     setup_commands = {
         "init": "archivebox.cli.archivebox_init.main",
@@ -69,6 +58,10 @@ class ArchiveBoxGroup(click.Group):
         # Introspection commands
         "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
     }
+    legacy_model_commands = {
+        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
+        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
+    }
     all_subcommands = {
         **meta_commands,
         **setup_commands,
@@ -80,29 +73,34 @@ class ArchiveBoxGroup(click.Group):
         "import": "add",
         "archive": "add",
     }
+    legacy_model_subcommands = {
+        "crawl": {"create", "list", "update", "delete"},
+        "snapshot": {"create", "list", "update", "delete"},
+    }
 
     @classmethod
     def get_canonical_name(cls, cmd_name):
         return cls.renamed_commands.get(cmd_name, cmd_name)
 
     @classmethod
-    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
-        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
-        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)
+    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
+        if cmd_name not in cls.legacy_model_commands:
+            return False
 
-    @classmethod
-    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
-        if not cls._needs_django_for_lazy_import(cmd_name):
-            return
+        try:
+            arg_idx = sys.argv.index(cmd_name)
+        except ValueError:
+            return False
 
-        from django.apps import apps
+        remaining_args = sys.argv[arg_idx + 1 :]
+        if not remaining_args:
+            return False
 
-        if apps.ready:
-            return
+        first_arg = remaining_args[0]
+        if first_arg in ("-h", "--help"):
+            return False
 
-        from archivebox.config.django import setup_django
-
-        setup_django()
+        return first_arg not in cls.legacy_model_subcommands[cmd_name]
 
     def get_command(self, ctx, cmd_name):
         # handle renamed commands
@@ -114,9 +112,11 @@ class ArchiveBoxGroup(click.Group):
             cmd_name = new_name
             ctx.invoked_subcommand = cmd_name
 
+        if self._should_use_legacy_model_command(cmd_name):
+            return self._lazy_load(self.legacy_model_commands[cmd_name])
+
         # handle lazy loading of commands
         if cmd_name in self.all_subcommands:
-            self._setup_django_for_lazy_import(cmd_name)
             return self._lazy_load(cmd_name)
 
         # fall-back to using click's default command lookup
@@ -129,12 +129,16 @@ class ArchiveBoxGroup(click.Group):
             import_path = cmd_name_or_path
         modname, funcname = import_path.rsplit(".", 1)
 
+        # print(f'LAZY LOADING {import_path}')
         mod = import_module(modname)
-        func = vars(mod)[funcname]
+        func = getattr(mod, funcname)
 
-        if func.__doc__ is None:
+        if not hasattr(func, "__doc__"):
             raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
 
+        # if not isinstance(cmd, click.BaseCommand):
+        # raise ValueError(f'lazy loading of {import_path} failed - not a click command')
+
         return func
 
 
@@ -155,48 +159,37 @@ def cli(ctx, help=False):
     # then we need to set up the django environment and check that we're in a valid data folder
     wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
     if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
+        # print('SETUP DJANGO AND CHECK DATA FOLDER')
         try:
-            # RUNSERVER_WORKER sets the web-worker markers on its own child.
-            # Setting them on the server parent leaks them through supervisord
-            # into runners waiting for takeover, which then register themselves
-            # as web workers and trigger spurious runner_watch reloads.
+            if subcommand == "server":
+                run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes")
+                if run_in_debug:
+                    os.environ["ARCHIVEBOX_RUNSERVER"] = "1"
+                    if "--reload" in sys.argv:
+                        os.environ["ARCHIVEBOX_AUTORELOAD"] = "1"
+
             from archivebox.config.django import setup_django
             from archivebox.misc.checks import check_data_folder, check_migrations
 
             setup_django()
-            if os.environ.get("ARCHIVEBOX_WANTS_INIT") == "1" and subcommand not in ("init", "help"):
-                # Universal `--init` was passed: build/upgrade the data folder before
-                # the regular preflight runs, so it succeeds on a fresh dir and an
-                # out-of-date schema both. Drop the env var afterwards so spawned
-                # subprocesses (supervisord workers, daphne, runner, etc.) inherit
-                # a clean env and don't re-trigger init in every child.
-                from archivebox.cli.archivebox_init import init as archivebox_init
-
-                archivebox_init(quick=True)
-                os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
             check_data_folder()
-            if subcommand != "update" and not check_migrations(auto_apply=True):
-                from archivebox.machine.models import Machine
-
-                # Host registration and config reconciliation belong to CLI
-                # startup; get_config() is also used by read-only HTTP views.
-                Machine.current()
+            if subcommand != "update":
+                check_migrations(auto_apply=True)
         except Exception as e:
             STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
             if subcommand not in ("manage", "shell"):  # not all management commands need django to be setup beforehand
                 raise
 
 
-def main(args=None, prog_name=None):
-    from archivebox.config.constants import CONSTANTS
-
-    os.environ.setdefault("ABX_PLUGINS_DIR", str(CONSTANTS.USER_PLUGINS_DIR))
-
+def main(args=None, prog_name=None, stdin=None):
     # show `docker run archivebox xyz` in help messages if running in docker
     IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
     IS_TTY = sys.stdin.isatty()
     prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
 
+    # stdin param allows passing input data from caller (used by __main__.py)
+    # currently not used by click-based CLI, but kept for backwards compatibility
+
     previous_unraisablehook = sys.unraisablehook
 
     def ignore_shutdown_unraisable(unraisable):
@@ -220,12 +213,6 @@ def main(args=None, prog_name=None):
         raise SystemExit(130) from None
     finally:
         sys.unraisablehook = previous_unraisablehook
-        supervisor_util = sys.modules.get("archivebox.workers.supervisord_util")
-        if supervisor_util is not None:
-            # Short commands can start a foreground-owned search daemon too.
-            # Finish its cleanup before another CLI can borrow a dying stack.
-            with redirect_stdout(sys.stderr):
-                supervisor_util.stop_own_supervisord_process()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_add.py

@@ -5,15 +5,14 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox add"
 
 import sys
+import json
 import os
-import signal
 from pathlib import Path
 
-from typing import Any, TYPE_CHECKING
+from typing import TYPE_CHECKING
 
 import rich_click as click
 
-from archivebox.config import CONSTANTS
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.util import parse_filesize_to_bytes
 
@@ -24,28 +23,21 @@ if TYPE_CHECKING:
     from archivebox.core.models import Snapshot
 
 
-def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
+def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
     from archivebox.misc.jsonl import read_args_or_stdin
-    from archivebox.misc.util import validate_url
 
     urls: list[str] = []
     for record in read_args_or_stdin(args):
         url = record.get("url")
         if isinstance(url, str) and url:
-            try:
-                urls.append(validate_url(url))
-            except ValueError as err:
-                raise click.BadParameter(str(err), param_hint="URL") from err
+            urls.append(url)
 
         urls_field = record.get("urls")
         if isinstance(urls_field, str):
             for line in urls_field.splitlines():
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    try:
-                        urls.append(validate_url(line))
-                    except ValueError as err:
-                        raise click.BadParameter(str(err), param_hint="URL") from err
+                    urls.append(line)
 
     return urls
 
@@ -66,12 +58,23 @@ def add(
     parser: str = "auto",
     plugins: str = "",
     persona: str = "Default",
+    overwrite: bool = False,
+    update: bool | None = None,
     index_only: bool = False,
     bg: bool = False,
     created_by_id: int | None = None,
-    config: dict[str, Any] | None = None,
 ) -> tuple[Crawl, QuerySet[Snapshot]]:
-    """Add a URL list or imported URL document to a new Crawl."""
+    """Add a new URL or list of URLs to your archive.
+
+    The flow is:
+    1. Save URLs to sources file
+    2. Create Crawl with URLs and max_depth
+    3. Crawl runner creates Snapshots from Crawl URLs (depth=0)
+    4. Crawl runner runs parser extractors on root snapshots
+    5. Parser extractors output to urls.jsonl
+    6. URLs are added to Crawl.urls and child Snapshots are created
+    7. Repeat until max_depth is reached
+    """
 
     from rich import print
 
@@ -80,14 +83,14 @@ def add(
     crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
     crawl_timeout = int(crawl_timeout or 0)
     snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
+    from archivebox import CONSTANTS
     from archivebox.config.permissions import USER, HOSTNAME
     from archivebox.config.common import get_config
 
-    config_overrides = dict(config or {})
-    runtime_config = get_config()
+    config = get_config()
     crawl_max_concurrent_snapshots_override = crawl_max_concurrent_snapshots is not None
     if crawl_max_concurrent_snapshots is None:
-        crawl_max_concurrent_snapshots = runtime_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
+        crawl_max_concurrent_snapshots = config.CRAWL_MAX_CONCURRENT_SNAPSHOTS
     crawl_max_concurrent_snapshots = int(crawl_max_concurrent_snapshots)
 
     if depth not in (0, 1, 2, 3, 4):
@@ -110,30 +113,33 @@ def add(
     from archivebox.misc.logging_util import printable_filesize
     from archivebox.misc.system import get_dir_size
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from django.contrib.auth import get_user_model
     from django.utils import timezone
 
     created_by_id = created_by_id or get_or_create_system_user_pk()
+    created_by = get_user_model().objects.filter(pk=created_by_id).first()
     started_at = timezone.now()
+    if update is None:
+        update = not config.ONLY_NEW
 
     if isinstance(urls, str):
-        source_text = urls
+        url_list = [line.strip() for line in urls.splitlines() if line.strip()]
     else:
-        from archivebox.misc.util import validate_url
+        url_list = [str(url).strip() for url in urls if str(url).strip()]
+    if snapshot_ids and len(snapshot_ids) != len(url_list):
+        raise ValueError("snapshot_ids length must match urls length")
 
-        source_text = "\n".join(validate_url(str(url)) for url in urls)
+    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
+    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
+    sources_file.parent.mkdir(parents=True, exist_ok=True)
+    if snapshot_ids:
+        sources_file.write_text(
+            "\n".join(json.dumps({"url": url, "id": snapshot_ids[index], "tags": tag, "depth": 0}) for index, url in enumerate(url_list)),
+        )
+    else:
+        sources_file.write_text("\n".join(url_list))
 
     # 2. Create a new Crawl with inline URLs
-    # Foreground add must claim runner ownership before publishing runnable
-    # work. Otherwise an existing server runner can lease the new Crawl in the
-    # gap between Crawl.objects.create() and current_command(), then be killed
-    # by the newer add process halfway through its first Snapshot hook.
-    command = None
-    if not bg and not index_only:
-        from archivebox.machine.models import Process
-        from archivebox.core.takeover_util import current_command
-
-        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR)
-
     cli_args = [*sys.argv]
     if cli_args[0].lower().endswith("archivebox"):
         cli_args[0] = "archivebox"
@@ -141,14 +147,19 @@ def add(
 
     timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
 
+    # Read URLs directly into crawl
+    urls_content = sources_file.read_text()
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
-    persona_obj = Persona.get_or_create_named(persona_name)
-    effective_persona_config = get_config(persona=persona_obj)
+    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
+    persona_obj.ensure_dirs()
+    effective_persona_config = get_config(persona=persona_obj, user=created_by)
 
     crawl_config = {
         "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
+        **({"ONLY_NEW": not update} if bool(not update) != bool(effective_persona_config.ONLY_NEW) else {}),
         **({"INDEX_ONLY": True} if index_only else {}),
+        **({"OVERWRITE": True} if overwrite else {}),
         **({"PLUGINS": plugins} if plugins else {}),
         **(
             {"CRAWL_MAX_CONCURRENT_SNAPSHOTS": crawl_max_concurrent_snapshots}
@@ -164,45 +175,30 @@ def add(
         **({"URL_ALLOWLIST": url_allowlist} if url_allowlist else {}),
         **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
     }
-    # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
-    # priority for crawl-frozen keys. Runtime-derived execution keys are
-    # stripped by Crawl.save() and rederived when hooks run.
-    crawl_config.update(config_overrides)
 
-    # Crawl.urls is the user's submitted source, not a derived work queue for
-    # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
-    # callers audit or resume imports without losing RSS/Netscape/JSON metadata
-    # that is not representable as one plain URL per line.
-    try:
-        crawl = Crawl.objects.create(
-            urls=source_text,
-            max_depth=depth,
-            tags_str=tag,
-            persona_id=persona_obj.id,
-            label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
-            created_by_id=created_by_id,
-            status=Crawl.StatusChoices.QUEUED,
-            retry_at=None if index_only else timezone.now(),
-            config=crawl_config,
-        )
-        first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
-    except BaseException:
-        if command is not None:
-            command.mark_exited(exit_code=1)
-        raise
-    if command is not None:
-        command.url = first_url
-        command.save(update_fields=["url", "modified_at"])
+    crawl = Crawl.objects.create(
+        urls=urls_content,
+        max_depth=depth,
+        tags_str=tag,
+        persona_id=persona_obj.id,
+        label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
+        created_by_id=created_by_id,
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=None if index_only else timezone.now(),
+        config=crawl_config,
+    )
 
     print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
+    first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
     print(f"    [dim]First URL: {first_url}[/dim]")
 
-    # The runner parses Crawl.urls after claiming the Crawl, then persists the
-    # resulting depth-0 Snapshot facts. Recursive discoveries start at depth 1.
+    # 3. The CrawlMachine will create Snapshots from all URLs when started
+    #    Parser extractors run on snapshots and discover more URLs
+    #    Discovered URLs become child Snapshots (depth+1)
 
     if index_only:
-        print("[yellow]\\[*] Index-only mode - URLs queued, runner not started[/yellow]")
-        return crawl, crawl.snapshot_set.none()
+        print("[yellow]\\[*] Index-only mode - crawl queued, runner not started[/yellow]")
+        return crawl, crawl.snapshot_set.all()
 
     # 5. Start the crawl runner to process the queue
     #    The runner will:
@@ -212,65 +208,32 @@ def add(
     #    - Repeat until max_depth reached
 
     if bg:
-        # Background mode: just queue work and return (background runner via server will pick it up).
+        # Background mode: just queue work and return (background runner via server will pick it up)
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
-        from archivebox.core.takeover_util import command_owns_foreground_runner, standby_until_foreground_runner_needed
-        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
+        from archivebox.machine.models import Process
+        from archivebox.services.supervision_service import command_owns_runtime_stack, current_command, standby_until_runtime_stack_needed
+        from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
 
-        assert command is not None
+        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
         exit_code = 0
-        interactive_interrupts = sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
-
-        def interrupt_hook() -> None:
-            raise KeyboardInterrupt
-
-        def crawl_is_complete() -> bool:
-            crawl.refresh_from_db(fields=["status"])
-            return crawl.status not in crawl.RUNNABLE_STATES
-
         try:
             try:
-                with (
-                    foreground_shutdown_signals(
-                        first_signal_message=None,
-                        interrupt_handlers={signal.SIGINT: interrupt_hook},
-                    ),
-                    foreground_parent_watchdog(),
-                ):
+                with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
                     while True:
-                        standby = standby_until_foreground_runner_needed(
-                            command,
-                            data_dir=CONSTANTS.DATA_DIR,
-                            work_is_complete=crawl_is_complete,
-                        )
-                        if standby["work_completed"]:
-                            break
+                        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
                         exit_code = run_runner_worker(
                             ["--crawl-id", str(crawl.id)],
                             name=f"worker_runner_add_{os.getpid()}",
-                            interactive_interrupts=interactive_interrupts,
-                            config=get_config(crawl=crawl),
+                            interactive_interrupts=True,
                         )
-                        crawl.refresh_from_db(fields=["status", "retry_at"])
-                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
-                            break
-                        # A shared supervisord can be stopped by another
-                        # foreground owner (for example `archivebox server`
-                        # shutting down) and stop the one-shot add runner while
-                        # this add's crawl is still runnable. Keep the add
-                        # process alive so it can re-enter the normal
-                        # foreground-runner ownership loop and finish its crawl.
-                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
-                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
-                            continue
                         if exit_code == 0:
                             break
-                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
+                        if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
                             continue
                         raise SystemExit(exit_code)
             except KeyboardInterrupt:
@@ -286,14 +249,13 @@ def add(
         # Print summary for foreground runs
         try:
             crawl.refresh_from_db()
+            snapshots_count = crawl.snapshot_set.count()
             try:
                 from django.db.models import Count, Sum
 
-                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("output_size"))
-                snapshots_count = int(totals["snapshot_count"] or 0)
-                total_bytes = int(totals["total_bytes"] or 0)
+                totals = crawl.snapshot_set.aggregate(snapshot_count=Count("id"), total_bytes=Sum("archiveresult__output_size"))
+                total_bytes = int(totals["total_bytes"] or 0) if totals["snapshot_count"] else 0
             except Exception:
-                snapshots_count = crawl.snapshot_set.count()
                 total_bytes, _, _ = get_dir_size(crawl.output_dir)
             total_size = printable_filesize(total_bytes)
             total_time = timezone.now() - started_at
@@ -314,17 +276,13 @@ def add(
             except Exception:
                 rel_output_str = str(crawl.output_dir)
 
-            from archivebox.core.routes_util import build_admin_url
+            from archivebox.core.host_utils import build_admin_url
 
             admin_url = build_admin_url(f"/admin/crawls/crawl/{crawl.id}/change/", config=config)
 
-            if snapshots_count:
-                print("\n[bold]snapshot output saved to:[/bold]")
-                for snapshot in crawl.snapshot_set.order_by("created_at").iterator():
-                    click.echo(f"  {snapshot.output_dir}")
-            print("\n[bold]crawl logs and setup:[/bold]")
-            click.echo(f"  {rel_output_str}")
-            click.echo(f"  {admin_url}")
+            print("\n[bold]crawl output saved to:[/bold]")
+            print(f"  {rel_output_str}")
+            print(f"  {admin_url}")
             print(f"\n[bold]total urls snapshotted:[/bold] {snapshots_count}")
             print(f"[bold]total size:[/bold] {total_size}")
             print(f"[bold]total time:[/bold] {duration_str}")
@@ -355,18 +313,10 @@ def add(
 @click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
 @click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
 @click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
-@click.option("--extract", default="", help="Comma-separated plugins or output types to extract e.g. title,pdf,text/html,image,...")
 @click.option("--persona", default="Default", help="Authentication profile to use when archiving")
-@click.option(
-    "--only-new/--no-only-new",
-    "only_new",
-    default=None,
-    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
-    "Pass --no-only-new to force re-archive of URLs that already exist.",
-)
+@click.option("--overwrite", "-F", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
+@click.option("--update", is_flag=True, default=None, help="Retry any previously skipped/failed URLs when re-adding them")
 @click.option("--index-only", is_flag=True, help="Just add the URLs to the index without archiving them now")
-@click.option("--overwrite", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
-@click.option("--update", is_flag=True, help="Re-archive URLs even if they already exist (alias for --no-only-new)")
 @click.option("--bg", is_flag=True, help="Run archiving in background (queue work and return immediately)")
 @click.argument("urls", nargs=-1, type=click.Path())
 @docstring(add.__doc__)
@@ -377,12 +327,7 @@ def main(**kwargs):
 
     with foreground_shutdown_signals(), foreground_parent_watchdog():
         raw_urls = kwargs.pop("urls")
-        if raw_urls:
-            urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
-        elif not sys.stdin.isatty():
-            urls = sys.stdin.read()
-        else:
-            urls = []
+        urls = _collect_input_urls(raw_urls)
         if not urls:
             raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
         if int(kwargs.get("max_urls") or 0) < 0:
@@ -400,46 +345,7 @@ def main(**kwargs):
         if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
             raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
 
-        # Translate --only-new/--no-only-new into a crawl config override.
-        # add() takes config overrides as a dict; no per-flag kwargs.
-        overwrite = kwargs.pop("overwrite", False)
-        update = kwargs.pop("update", False)
-        only_new = kwargs.pop("only_new", None)
-        extract = kwargs.pop("extract", "")
-        if overwrite or update:
-            only_new = False
-        if only_new is not None:
-            kwargs["config"] = {"ONLY_NEW": bool(only_new)}
-        if extract:
-            from archivebox.plugins.discovery import get_plugin_catalog
-
-            catalog = get_plugin_catalog()
-            tokens = [token.strip() for token in extract.split(",") if token.strip()]
-            plugin_names = {name.lower(): name for name in catalog}
-            selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
-            selected += catalog.matching_output(tokens)
-            if not selected:
-                raise click.UsageError(f"No plugins found matching extract types: {extract}")
-            existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
-            kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
-
-        from archivebox.machine.models import Process
-
-        # CLI commands own their process record and logs. Queue model saves
-        # must not register a process implicitly: HTTP add() shares that path.
-        process = Process.current()
-        exit_code = 0
-        try:
-            add(urls=urls, **kwargs)
-        except BaseException:
-            exit_code = 1
-            raise
-        finally:
-            # --bg/--index-only return without the foreground runner's cleanup.
-            # Leaving them RUNNING makes the active-process index accumulate
-            # historical CLI commands forever, defeating bounded recovery.
-            if process.status == Process.StatusChoices.RUNNING:
-                process.mark_exited(exit_code=exit_code)
+        add(urls=urls, **kwargs)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_archiveresult.py

@@ -6,13 +6,13 @@ archivebox archiveresult <action> [args...] [--filters]
 Manage ArchiveResult records (plugin extraction results).
 
 Actions:
-    create  - Emit plugin extraction request records for Snapshots
+    create  - Create ArchiveResults for Snapshots (queue extractions)
     list    - List ArchiveResults as JSONL (with optional filters)
     update  - Update ArchiveResults from stdin JSONL
     delete  - Delete ArchiveResults from stdin JSONL
 
 Examples:
-    # Emit extraction requests; `archivebox run` schedules their parent snapshots
+    # Create ArchiveResults for snapshots (queue for extraction)
     archivebox snapshot list --status=queued | archivebox archiveresult create
     archivebox archiveresult create --plugin=screenshot --snapshot-id=<uuid>
 
@@ -20,6 +20,9 @@ Examples:
     archivebox archiveresult list --status=failed
     archivebox archiveresult list --plugin=screenshot --status=succeeded
 
+    # Update (reset failed extractions to queued)
+    archivebox archiveresult list --status=failed | archivebox archiveresult update --status=queued
+
     # Delete
     archivebox archiveresult list --plugin=singlefile | archivebox archiveresult delete --yes
 
@@ -35,7 +38,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_util import apply_filters
+from archivebox.cli.cli_utils import apply_filters
 
 
 def build_archiveresult_request(snapshot_id: str, plugin: str, hook_name: str = "", status: str = "queued") -> dict:
@@ -71,7 +74,7 @@ def create_archiveresults(
         1: Failure
     """
     from archivebox.config.common import get_config
-    from archivebox.plugins.hooks import discover_hooks
+    from archivebox.hooks import discover_hooks
     from archivebox.misc.jsonl import read_stdin, write_record, TYPE_SNAPSHOT, TYPE_ARCHIVERESULT
     from archivebox.core.models import Snapshot
 
@@ -141,20 +144,21 @@ def create_archiveresults(
 
     created_count = 0
     for snapshot in snapshots:
-        config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
-        hooks = [
-            hook
-            for hook in discover_hooks("Snapshot", filter_disabled=not plugin, config=config)
-            if not plugin or hook.parent.name == plugin
-        ]
-        for hook_path in hooks:
-            hook_name = hook_path.stem
-            plugin_name = hook_path.parent.name
+        if plugin:
             if not is_tty:
-                write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
+                write_record(build_archiveresult_request(snapshot.id, plugin, status=status))
             created_count += 1
+        else:
+            config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
+            hooks = discover_hooks("Snapshot", config=config)
+            for hook_path in hooks:
+                hook_name = hook_path.stem
+                plugin_name = hook_path.parent.name
+                if not is_tty:
+                    write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
+                created_count += 1
 
-    rprint(f"[green]Created {created_count} extraction request records[/green]", file=sys.stderr)
+    rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
     return 0
 
 
@@ -338,7 +342,7 @@ def main():
 @click.option("--plugin", "-p", help="Plugin name (e.g., screenshot, singlefile)")
 @click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
 def create_cmd(snapshot_id: str | None, plugin: str | None, status: str):
-    """Emit Snapshot plugin extraction requests as JSONL."""
+    """Create ArchiveResults for Snapshots from stdin JSONL."""
     sys.exit(create_archiveresults(snapshot_id=snapshot_id, plugin=plugin, status=status))
 
 

archivebox/cli/archivebox_binary.py

@@ -33,7 +33,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_util import apply_filters
+from archivebox.cli.cli_utils import apply_filters
 
 
 # =============================================================================

archivebox/cli/archivebox_config.py

@@ -6,7 +6,6 @@ import sys
 import toml
 import rich_click as click
 from rich import print
-from pathlib import Path
 
 from archivebox.misc.util import docstring, enforce_types
 from archivebox.misc.toml_util import CustomTOMLEncoder
@@ -31,17 +30,12 @@ def config(
     from archivebox.misc.logging_util import printable_config
     from abx_plugins.plugins.base.utils import resolve_alias
     from archivebox.config.collection import write_config_file
-    from archivebox.config import CONSTANTS_CONFIG
     from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
-    from archivebox.plugins.discovery import discover_plugin_configs
+    from archivebox.hooks import discover_plugin_configs
 
     check_data_folder()
 
     FLAT_CONFIG = get_config().as_dict()
-    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
-    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
-    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
-    readable_config = {**writable_config, **readonly_config}
     CONFIGS = get_all_configs()
     plugin_schemas = {
         plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
@@ -59,20 +53,23 @@ def config(
     matching_config = {}
     if search:
         if config_options:
-            search_terms = [key.strip().lower() for key in config_options]
+            config_options = [
+                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
+            ]
+            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            for config_section in CONFIGS.values():
+                aliases = getattr(config_section, "aliases", {})
 
-            for existing_key, value in readable_config.items():
-                if any(term in existing_key.lower() or term in str(value).lower() for term in search_terms):
-                    matching_config[existing_key] = value
+                for search_key in config_options:
+                    # search all aliases in the section
+                    for alias_key, key in aliases.items():
+                        if search_key.lower() in alias_key.lower():
+                            matching_config[key] = dict(config_section)[key]
 
-            for alias, key in core_config_aliases.items():
-                if key in readable_config and any(term in alias.lower() for term in search_terms):
-                    matching_config[key] = readable_config[key]
-
-            for schema in plugin_schemas.values():
-                for key, metadata in schema.items():
-                    if key in readable_config and any(term in key.lower() or term in str(metadata).lower() for term in search_terms):
-                        matching_config[key] = readable_config[key]
+                    # search all keys and values in the section
+                    for existing_key, value in dict(config_section).items():
+                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
+                            matching_config[existing_key] = value
 
         print(printable_config(matching_config))
         raise SystemExit(not matching_config)
@@ -82,18 +79,18 @@ def config(
             config_options = [
                 core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
             ]
-            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
-            failed_config = [key for key in config_options if key not in readable_config]
+            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
+            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
             if failed_config:
                 print("\n[red][X] These options failed to get[/red]")
                 print("    {}".format("\n    ".join(config_options)))
                 raise SystemExit(1)
         else:
-            matching_config = readable_config
+            matching_config = FLAT_CONFIG
 
         # Display core config sections
         for config_section in CONFIGS.values():
-            section_header = config_section.toml_section_header
+            section_header = getattr(config_section, "toml_section_header", "")
             if isinstance(section_header, str) and section_header:
                 print(f"[grey53]\\[{section_header}][/grey53]")
             else:
@@ -103,18 +100,12 @@ def config(
             print(_format_toml(kv_in_section))
             print("[grey53]################################################################[/grey53]")
 
-        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
-        if readonly_keys:
-            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
-            print(_format_toml(readonly_keys))
-            print("[grey53]################################################################[/grey53]")
-
         plugin_keys = {}
 
         # Collect all plugin config keys
         for schema in plugin_schemas.values():
             for key in schema.keys():
-                if key in matching_config and key in writable_config:
+                if key in matching_config:
                     plugin_keys[key] = matching_config[key]
 
         # Display all plugin config in single [PLUGINS] section
@@ -144,7 +135,7 @@ def config(
                     f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
                 )
 
-            if key in writable_config:
+            if key in FLAT_CONFIG:
                 new_config[key] = val.strip()
             else:
                 failed_options.append(line)

archivebox/cli/archivebox_crawl.py

@@ -39,7 +39,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_util import apply_filters
+from archivebox.cli.cli_utils import apply_filters
 
 
 # =============================================================================
@@ -65,7 +65,6 @@ def create_crawl(
         1: Failure
     """
     from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
-    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
 
@@ -99,11 +98,7 @@ def create_crawl(
         # Collect URLs
         url = record.get("url")
         if url:
-            try:
-                url_list.append(validate_url(str(url)))
-            except ValueError as err:
-                rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
-                return 1
+            url_list.append(url)
 
         # Handle 'urls' field (newline-separated)
         urls_field = record.get("urls")
@@ -111,11 +106,7 @@ def create_crawl(
             for line in urls_field.split("\n"):
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    try:
-                        url_list.append(validate_url(line))
-                    except ValueError as err:
-                        rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
-                        return 1
+                    url_list.append(line)
 
     # Output pass-through records first
     if not is_tty:

archivebox/cli/archivebox_crawl_compat.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox crawl"
+
+import sys
+
+import rich_click as click
+
+from archivebox.cli.archivebox_add import add
+
+
+@click.command(context_settings={"ignore_unknown_options": True})
+@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
+@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
+@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
+@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
+@click.argument("urls", nargs=-1)
+def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
+    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
+    del status, wait
+    add(list(urls), depth=depth, tag=tag, bg=True)
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

archivebox/cli/archivebox_extract.py

@@ -1,135 +1,106 @@
 #!/usr/bin/env python3
 
-"""Run abx-dl snapshot hooks for existing ArchiveBox snapshots."""
+"""
+archivebox extract [snapshot_ids...] [--plugins=NAMES]
+
+Run plugins on Snapshots. Accepts snapshot IDs as arguments, from stdin, or via JSONL.
+
+Input formats:
+    - Snapshot UUIDs (one per line)
+    - JSONL: {"type": "Snapshot", "id": "...", "url": "..."}
+    - JSONL: {"type": "ArchiveResult", "snapshot_id": "...", "plugin": "..."}
+
+Output (JSONL):
+    {"type": "ArchiveResult", "id": "...", "snapshot_id": "...", "plugin": "...", "status": "..."}
+
+Examples:
+    # Extract specific snapshot
+    archivebox extract 01234567-89ab-cdef-0123-456789abcdef
+
+    # Pipe from snapshot command
+    archivebox snapshot https://example.com | archivebox extract
+
+    # Run specific plugins only
+    archivebox extract --plugins=screenshot,singlefile 01234567-89ab-cdef-0123-456789abcdef
+
+    # Chain commands
+    archivebox crawl https://example.com | archivebox snapshot | archivebox extract
+"""
 
 __package__ = "archivebox.cli"
 __command__ = "archivebox extract"
 
 import sys
 from collections import defaultdict
-from contextlib import redirect_stdout
+from itertools import product
 
 import rich_click as click
 
 
-def _resolve_requests(records: list[dict], plugins: str) -> tuple[dict[str, set[str]], set[str]]:
-    """Resolve CLI records to snapshot-level execution requests.
-
-    ArchiveResult input is accepted as a convenient reference to its parent
-    Snapshot and plugin. It is never reset or converted into queued work.
+def process_archiveresult_by_id(archiveresult_id: str) -> int:
     """
-    from archivebox.api.v1_core import _uuid_ref_query
-    from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.misc.jsonl import TYPE_ARCHIVERESULT
+    Re-run extraction for a single ArchiveResult by ID.
 
-    explicit_plugins = {name.strip() for name in plugins.split(",") if name.strip()}
-    requested: dict[str, set[str]] = defaultdict(set)
-    missing: set[str] = set()
-
-    for record in records:
-        record_type = record.get("type")
-        record_id = str(record.get("id") or "")
-
-        if record_type == TYPE_ARCHIVERESULT and record.get("snapshot_id"):
-            requested[str(record["snapshot_id"])].update(explicit_plugins or {str(record.get("plugin") or "")})
-            requested[str(record["snapshot_id"])].discard("")
-            continue
-
-        if record_type == TYPE_ARCHIVERESULT:
-            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
-            if result is not None:
-                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
-                continue
-
-        snapshot_id = str(record.get("snapshot_id") or record_id or "")
-        snapshot = Snapshot.objects.filter(id=snapshot_id).only("id").first() if snapshot_id else None
-        if snapshot is None and record.get("url"):
-            snapshot = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").only("id").first()
-        # Bare UUID CLI arguments are parsed as Snapshot records because the
-        # input layer cannot know which model owns them. Preserve the public
-        # convenience of passing an ArchiveResult ID by trying that reference
-        # only after the Snapshot lookup misses.
-        if snapshot is None and record_id and not record.get("url"):
-            result = ArchiveResult.objects.filter(_uuid_ref_query("id", record_id)).only("snapshot_id", "plugin").first()
-            if result is not None:
-                requested[str(result.snapshot_id)].update(explicit_plugins or {result.plugin})
-                continue
-        if snapshot is None:
-            missing.add(snapshot_id or str(record.get("url") or ""))
-            continue
-        requested[str(snapshot.id)].update(explicit_plugins)
-
-    return requested, missing
-
-
-def _run_snapshot_requests(requested: dict[str, set[str]], *, wait: bool, show_progress: bool) -> int:
-    from django.utils import timezone
+    ArchiveResults are projected status rows, not queued work items. Re-running
+    a single result means resetting that row and queueing its parent snapshot
+    through the shared crawl runner with the corresponding plugin selected.
+    """
     from rich import print as rprint
-
-    from archivebox.core.models import Snapshot
+    from django.utils import timezone
+    from archivebox.core.models import ArchiveResult
     from archivebox.services.runner import run_crawl
 
-    snapshots = {str(snapshot.id): snapshot for snapshot in Snapshot.objects.filter(id__in=requested).select_related("crawl")}
-    if not snapshots:
-        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
+    try:
+        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
+    except ArchiveResult.DoesNotExist:
+        rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
         return 1
 
-    from archivebox.config.common import get_config
-    from archivebox.plugins.discovery import get_enabled_plugins
+    rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
 
-    for snapshot_id, plugin_names in requested.items():
-        snapshot = snapshots.get(snapshot_id)
-        if snapshot is not None and not plugin_names:
-            plugin_names.update(get_enabled_plugins(config=get_config(crawl=snapshot.crawl, snapshot=snapshot)))
+    try:
+        was_paused = archiveresult.snapshot.is_paused
+        archiveresult.reset_for_retry()
+        snapshot = archiveresult.snapshot
+        if not was_paused:
+            snapshot.queue_for_extraction()
+        else:
+            # A paused snapshot may still accept explicit maintenance for one
+            # ArchiveResult, but this path must not transition it back to
+            # queued/startable work.
+            snapshot.safe_update({"retry_at": timezone.now()}, refresh=False)
+        crawl = snapshot.crawl
+        if not crawl.claim_processing_lock(lock_seconds=10):
+            rprint(
+                f"[yellow]Crawl {crawl.id} is already owned by another runner[/yellow]",
+                file=sys.stderr,
+            )
+            return 1
 
-    if wait and any("search_backend_sonic" in plugin_names for plugin_names in requested.values()):
-        from archivebox.core.takeover_util import ensure_daemon_stack
+        try:
+            run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        finally:
+            if was_paused:
+                snapshot.restore_paused_scheduler_marker()
+        archiveresult.refresh_from_db()
 
-        with redirect_stdout(sys.stderr):
-            ensure_daemon_stack(reason="Sonic snapshot indexing")
+        if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
+            print(f"[green]Extraction succeeded: {archiveresult.output_str}[/green]")
+            return 0
+        elif archiveresult.status == ArchiveResult.StatusChoices.NORESULTS:
+            print(f"[dim]Extraction completed with no results: {archiveresult.output_str}[/dim]")
+            return 0
+        elif archiveresult.status == ArchiveResult.StatusChoices.FAILED:
+            print(f"[red]Extraction failed: {archiveresult.output_str}[/red]", file=sys.stderr)
+            return 1
+        else:
+            # Still in progress or backoff - not a failure
+            print(f"[yellow]Extraction status: {archiveresult.status}[/yellow]")
+            return 0
 
-    # Explicit extraction resumes open/paused snapshots at the snapshot level.
-    # Sealed snapshots stay sealed during targeted maintenance backfills.
-    if wait:
-        for snapshot in snapshots.values():
-            if snapshot.status != Snapshot.StatusChoices.SEALED:
-                snapshot.update_and_requeue(
-                    status=Snapshot.StatusChoices.QUEUED,
-                    retry_at=timezone.now(),
-                )
-
-    if not wait:
-        for snapshot_id, plugin_names in requested.items():
-            snapshot = snapshots.get(snapshot_id)
-            if snapshot is None:
-                continue
-            if plugin_names:
-                snapshot.schedule_plugin_run(plugin_names, when=timezone.now())
-            elif snapshot.status == Snapshot.StatusChoices.SEALED:
-                snapshot.update_and_requeue(retry_at=timezone.now())
-            else:
-                snapshot.update_and_requeue(
-                    status=Snapshot.StatusChoices.QUEUED,
-                    retry_at=timezone.now(),
-                )
-        if show_progress:
-            rprint(f"[blue]Queued {len(snapshots)} snapshots for extraction[/blue]", file=sys.stderr)
-        return 0
-
-    grouped: dict[tuple[str, tuple[str, ...]], list[str]] = defaultdict(list)
-    for snapshot_id, plugin_names in requested.items():
-        snapshot = snapshots.get(snapshot_id)
-        if snapshot is not None:
-            grouped[(str(snapshot.crawl_id), tuple(sorted(plugin_names)))].append(snapshot_id)
-
-    for (crawl_id, plugin_names), snapshot_ids in grouped.items():
-        run_crawl(
-            crawl_id,
-            snapshot_ids=sorted(snapshot_ids),
-            selected_plugins=list(plugin_names) or None,
-            show_progress=show_progress,
-        )
-    return 0
+    except Exception as e:
+        print(f"[red]Extraction error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+        return 1
 
 
 def run_plugins(
@@ -140,63 +111,306 @@ def run_plugins(
     emit_results: bool = True,
     show_progress: bool = True,
 ) -> int:
-    """Execute selected plugins through the snapshot-level runner."""
+    """
+    Run plugins on Snapshots from input.
+
+    Reads Snapshot IDs or JSONL from args/stdin, runs plugins, outputs JSONL.
+
+    Exit codes:
+        0: Success
+        1: Failure
+    """
     from rich import print as rprint
+    from django.utils import timezone
 
+    from archivebox.misc.jsonl import (
+        read_args_or_stdin,
+        write_record,
+        TYPE_SNAPSHOT,
+        TYPE_ARCHIVERESULT,
+    )
     from archivebox.core.models import Snapshot
-    from archivebox.misc.jsonl import read_args_or_stdin, write_record
+    from archivebox.core.models import ArchiveResult
+    from archivebox.services.runner import run_crawl
+    from abx_dl.models import discover_plugins
 
+    is_tty = sys.stdout.isatty()
+
+    # Parse comma-separated plugins list once (reused in creation and filtering)
+    plugins_list = [p.strip() for p in plugins.split(",") if p.strip()] if plugins else []
+
+    # Parse stdin/args exactly once per CLI invocation.
+    # `main()` may already have consumed stdin to distinguish Snapshot input from
+    # ArchiveResult IDs; if so, it must pass the parsed records through here
+    # instead of asking this helper to reread an already-drained pipe.
     if records is None:
         records = list(read_args_or_stdin(args))
+
     if not records:
         rprint("[yellow]No snapshots provided. Pass snapshot IDs as arguments or via stdin.[/yellow]", file=sys.stderr)
         return 1
 
-    requested, missing = _resolve_requests(records, plugins)
-    for value in sorted(missing):
-        rprint(f"[yellow]Snapshot or ArchiveResult not found: {value}[/yellow]", file=sys.stderr)
-    if not requested:
+    # Gather snapshot IDs and optional plugin constraints to process
+    snapshot_ids = set()
+    requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set)
+    for record in records:
+        record_type = record.get("type")
+
+        if record_type == TYPE_SNAPSHOT:
+            snapshot_id = record.get("id")
+            if snapshot_id:
+                snapshot_ids.add(str(snapshot_id))
+            elif record.get("url"):
+                # Look up by URL (get most recent if multiple exist)
+                snap = Snapshot.objects.filter(url=record["url"]).order_by("-created_at").first()
+                if snap:
+                    snapshot_ids.add(str(snap.id))
+                else:
+                    rprint(f"[yellow]Snapshot not found for URL: {record['url']}[/yellow]", file=sys.stderr)
+
+        elif record_type == TYPE_ARCHIVERESULT:
+            snapshot_id = record.get("snapshot_id")
+            if snapshot_id:
+                snapshot_ids.add(str(snapshot_id))
+                plugin_name = record.get("plugin")
+                if plugin_name and not plugins_list:
+                    requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
+
+        elif "id" in record:
+            # Assume it's a snapshot ID
+            snapshot_ids.add(str(record["id"]))
+
+    if not snapshot_ids:
+        rprint("[red]No valid snapshot IDs found in input[/red]", file=sys.stderr)
         return 1
 
-    exit_code = _run_snapshot_requests(requested, wait=wait, show_progress=show_progress)
-    if exit_code or not emit_results:
-        return exit_code
+    existing_snapshots = list(Snapshot.objects.filter(id__in=snapshot_ids).values_list("id", "crawl_id"))
+    existing_snapshot_ids = {str(snapshot_id) for snapshot_id, _crawl_id in existing_snapshots}
+    existing_crawl_ids = {str(crawl_id) for _snapshot_id, crawl_id in existing_snapshots}
+    missing_snapshot_ids = sorted(str(snapshot_id) for snapshot_id in snapshot_ids - existing_snapshot_ids)
+    for snapshot_id in missing_snapshot_ids:
+        rprint(f"[yellow]Snapshot {snapshot_id} not found[/yellow]", file=sys.stderr)
+
+    # Queue only the target plugin rows. Bulk updates keep large reindex runs
+    # from doing one SELECT+UPDATE per snapshot/plugin before hooks even start.
+    requested_pairs: set[tuple[str, str]] = set()
+    if plugins_list:
+        requested_pairs.update((snapshot_id, plugin_name) for snapshot_id, plugin_name in product(existing_snapshot_ids, plugins_list))
+    else:
+        requested_pairs.update(
+            (snapshot_id, plugin_name)
+            for snapshot_id, plugin_names in requested_plugins_by_snapshot.items()
+            if snapshot_id in existing_snapshot_ids
+            for plugin_name in plugin_names
+        )
+    plugins_by_name = discover_plugins()
+    requested_rows: set[tuple[str, str, str]] = set()
+    for snapshot_id, plugin_name in requested_pairs:
+        plugin = plugins_by_name.get(plugin_name)
+        hooks = plugin.filter_hooks("Snapshot") if plugin is not None else []
+        if hooks:
+            requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks)
+        else:
+            requested_rows.add((snapshot_id, plugin_name, ""))
+
+    reset_fields = {
+        "status": ArchiveResult.StatusChoices.QUEUED,
+        "output_str": "",
+        "output_json": None,
+        "output_files": {},
+        "output_size": 0,
+        "output_mimetypes": "",
+        "start_ts": None,
+        "end_ts": None,
+        "modified_at": timezone.now(),
+    }
+    if plugins_list:
+        ArchiveResult.objects.filter(snapshot_id__in=existing_snapshot_ids, plugin__in=plugins_list).update(**reset_fields)
+    elif requested_plugins_by_snapshot:
+        snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set)
+        for snapshot_id, plugin_names in requested_plugins_by_snapshot.items():
+            if snapshot_id in existing_snapshot_ids:
+                for plugin_name in plugin_names:
+                    snapshot_ids_by_plugin[plugin_name].add(snapshot_id)
+        for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(**reset_fields)
+    existing_rows = set(
+        ArchiveResult.objects.filter(
+            snapshot_id__in=existing_snapshot_ids,
+            plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+        ).values_list("snapshot_id", "plugin", "hook_name"),
+    )
+    missing_rows = requested_rows - {(str(snapshot_id), plugin_name, hook_name) for snapshot_id, plugin_name, hook_name in existing_rows}
+    if missing_rows:
+        ArchiveResult.objects.bulk_create(
+            [
+                ArchiveResult(
+                    snapshot_id=snapshot_id,
+                    plugin=plugin_name,
+                    hook_name=hook_name,
+                    status=ArchiveResult.StatusChoices.QUEUED,
+                )
+                for snapshot_id, plugin_name, hook_name in sorted(missing_rows)
+            ],
+            batch_size=500,
+        )
+
+    processed_count = len(existing_snapshot_ids)
+    queue_at = timezone.now()
+    if existing_snapshot_ids:
+        if requested_rows:
+            # Targeted ArchiveResult retries use retry_at as the scheduling
+            # signal and keep sealed snapshots sealed so extractors are not
+            # re-run outside the explicitly queued plugin rows. Paused snapshots
+            # also keep status=paused here: `retry_at` only asks the orchestrator
+            # to process the queued plugin rows, and run_due_snapshot restores
+            # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
+            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "modified_at"):
+                snapshot.safe_update({"retry_at": queue_at, "modified_at": queue_at}, refresh=False)
+        else:
+            # No plugin rows were requested, so this is a full snapshot retry.
+            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):
+                snapshot.safe_update(
+                    {
+                        "status": Snapshot.StatusChoices.QUEUED,
+                        "retry_at": queue_at,
+                        "current_step": 0,
+                        "modified_at": queue_at,
+                    },
+                    refresh=False,
+                )
+    if existing_crawl_ids and not requested_rows:
+        from archivebox.crawls.models import Crawl
+
+        for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"):
+            update_fields = {
+                "retry_at": queue_at,
+                "modified_at": queue_at,
+            }
+            if crawl.status != Crawl.StatusChoices.STARTED:
+                update_fields["status"] = Crawl.StatusChoices.QUEUED
+            crawl.safe_update(update_fields, refresh=False)
+
+    if processed_count == 0:
+        rprint("[red]No snapshots to process[/red]", file=sys.stderr)
+        return 1
+
+    if show_progress:
+        rprint(f"[blue]Queued {processed_count} snapshots for extraction[/blue]", file=sys.stderr)
+
+    # Run orchestrator if --wait (default)
+    if wait:
+        if show_progress:
+            rprint("[blue]Running plugins...[/blue]", file=sys.stderr)
+        snapshot_ids_by_crawl: dict[str, set[str]] = defaultdict(set)
+        for snapshot_id, crawl_id in existing_snapshots:
+            snapshot_ids_by_crawl[str(crawl_id)].add(str(snapshot_id))
+
+        for crawl_id, crawl_snapshot_ids in snapshot_ids_by_crawl.items():
+            from archivebox.crawls.models import Crawl
+
+            crawl = Crawl.objects.get(id=crawl_id)
+            if not crawl.claim_processing_lock(lock_seconds=10):
+                rprint(
+                    f"[yellow]Crawl {crawl_id} is already owned by another runner[/yellow]",
+                    file=sys.stderr,
+                )
+                return 1
+            selected_plugins = (
+                plugins_list
+                or sorted(
+                    {plugin for snapshot_id in crawl_snapshot_ids for plugin in requested_plugins_by_snapshot.get(str(snapshot_id), set())},
+                )
+                or None
+            )
+            run_crawl(
+                crawl_id,
+                snapshot_ids=sorted(crawl_snapshot_ids),
+                selected_plugins=selected_plugins,
+                show_progress=show_progress,
+            )
+
+    if not emit_results:
+        return 0
+
+    # Output results as JSONL (when piped) or human-readable (when TTY)
+    for snapshot_id in snapshot_ids:
+        try:
+            snapshot = Snapshot.objects.get(id=snapshot_id)
+            results = snapshot.archiveresult_set.all()
+            if plugins_list:
+                results = results.filter(plugin__in=plugins_list)
+
+            for result in results:
+                if is_tty:
+                    status_color = {
+                        "succeeded": "green",
+                        "failed": "red",
+                        "skipped": "yellow",
+                    }.get(result.status, "dim")
+                    rprint(
+                        f"  [{status_color}]{result.status}[/{status_color}] {result.plugin} → {result.output_str or ''}",
+                        file=sys.stderr,
+                    )
+                else:
+                    write_record(result.to_json())
+        except Snapshot.DoesNotExist:
+            continue
 
-    is_tty = sys.stdout.isatty()
-    for snapshot in Snapshot.objects.filter(id__in=requested):
-        results = snapshot.archiveresult_set.all()
-        requested_plugins = requested[str(snapshot.id)]
-        if requested_plugins:
-            results = results.filter(plugin__in=requested_plugins)
-        for result in results:
-            if is_tty:
-                color = {"succeeded": "green", "failed": "red", "skipped": "yellow"}.get(result.status, "dim")
-                rprint(f"  [{color}]{result.status}[/{color}] {result.plugin} → {result.output_str or ''}", file=sys.stderr)
-            else:
-                write_record(result.to_json())
     return 0
 
 
-def process_archiveresult_by_id(archiveresult_id: str) -> int:
-    """Re-run the parent Snapshot plugin referenced by an ArchiveResult."""
-    return run_plugins((), records=[{"id": archiveresult_id}], wait=True)
+def is_archiveresult_id(value: str) -> bool:
+    """Check if value looks like an ArchiveResult UUID."""
+    import re
+
+    uuid_pattern = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
+    if not uuid_pattern.match(value):
+        return False
+    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
+    from archivebox.core.models import ArchiveResult
+
+    return ArchiveResult.objects.filter(id=value).exists()
 
 
 @click.command()
-@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run")
+@click.option("--plugins", "--plugin", "-p", default="", help="Comma-separated list of plugins to run (e.g., screenshot,singlefile)")
 @click.option("--wait/--no-wait", default=True, help="Wait for plugins to complete (default: wait)")
 @click.argument("args", nargs=-1)
 def main(plugins: str, wait: bool, args: tuple):
-    """Run plugins on Snapshots; ArchiveResult IDs select their parent plugin."""
+    """Run plugins on Snapshots, or process existing ArchiveResults by ID"""
     from archivebox.misc.jsonl import read_args_or_stdin
 
+    # Read all input
     records = list(read_args_or_stdin(args))
+
     if not records:
         from rich import print as rprint
 
         rprint("[yellow]No Snapshot IDs or ArchiveResult IDs provided. Pass as arguments or via stdin.[/yellow]", file=sys.stderr)
         sys.exit(1)
-    sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
+
+    # Check if input looks like existing ArchiveResult IDs to process
+    all_are_archiveresult_ids = all(is_archiveresult_id(r.get("id") or r.get("url", "")) for r in records)
+
+    if all_are_archiveresult_ids:
+        # Process existing ArchiveResults by ID
+        from rich import print as rprint
+
+        exit_code = 0
+        for record in records:
+            archiveresult_id = record.get("id") or record.get("url")
+            if not isinstance(archiveresult_id, str):
+                rprint(f"[red]Invalid ArchiveResult input: {record}[/red]", file=sys.stderr)
+                exit_code = 1
+                continue
+            result = process_archiveresult_by_id(archiveresult_id)
+            if result != 0:
+                exit_code = result
+        sys.exit(exit_code)
+    else:
+        # Default behavior: run plugins on Snapshots from input
+        sys.exit(run_plugins(args, records=records, plugins=plugins, wait=wait))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_help.py

@@ -5,8 +5,6 @@ __command__ = "archivebox help"
 import ast
 import importlib.util
 import os
-import sys
-from datetime import UTC, datetime
 from pathlib import Path
 
 import click
@@ -15,10 +13,6 @@ from rich.panel import Panel
 
 
 def _command_doc(cmd: str, import_path: str) -> str:
-    def first_doc_line(docstring: str | None) -> str:
-        lines = (docstring or "").splitlines()
-        return lines[0] if lines else ""
-
     modname, _ = import_path.rsplit(".", 1)
     spec = importlib.util.find_spec(modname)
     if spec is None or spec.origin is None:
@@ -32,9 +26,9 @@ def _command_doc(cmd: str, import_path: str) -> str:
     for name in (cmd, "main"):
         for node in tree.body:
             if isinstance(node, ast.FunctionDef) and node.name == name:
-                return first_doc_line(ast.get_docstring(node))
+                return (ast.get_docstring(node) or "").splitlines()[0]
 
-    return first_doc_line(ast.get_docstring(tree))
+    return (ast.get_docstring(tree) or "").splitlines()[0]
 
 
 def help() -> None:
@@ -42,14 +36,11 @@ def help() -> None:
 
     from archivebox.cli import ArchiveBoxGroup
     from archivebox.config import CONSTANTS
+    from archivebox.config.common import get_config
     from archivebox.config.permissions import IN_DOCKER
+    from archivebox.misc.logging_util import log_cli_command
 
-    version_msg = (
-        f"[dark_magenta]\\[{datetime.now(UTC):%Y-%m-%d %H:%M:%S}][/dark_magenta] "
-        f"[dark_red]ArchiveBox[/dark_red] [dark_goldenrod]v{CONSTANTS.VERSION}[/dark_goldenrod]: "
-        "[green4]archivebox [green3]help"
-    )
-    print(Panel(version_msg), file=sys.stderr)
+    log_cli_command("help", [], None, ".")
 
     COMMANDS_HELP_TEXT = (
         "\n    ".join(f"[green]{cmd.ljust(20)}[/green] {_command_doc(cmd, path)}" for cmd, path in ArchiveBoxGroup.meta_commands.items())
@@ -68,7 +59,7 @@ def help() -> None:
     [blue]docker compose run[/blue] [dark_green]archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
 
     [grey53]# using Docker:[/grey53]
-    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 5797:5797[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
+    [blue]docker run[/blue] -v [light_slate_blue]$PWD:/data[/light_slate_blue] [grey53]-p 8000:8000[/grey53] -it [dark_green]archivebox/archivebox[/dark_green] [green]\\[command][/green] [green3][...args][/green3] [violet][--help][/violet] [grey53][--version][/grey53]
 """
         if IN_DOCKER
         else ""
@@ -94,7 +85,8 @@ def help() -> None:
     [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
 """)
 
-    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
+    config = get_config()
+    if os.access(config.ARCHIVE_DIR, os.R_OK) and config.ARCHIVE_DIR.is_dir():
         pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
         EXAMPLE_USAGE = f"""
 [light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]
@@ -109,7 +101,7 @@ def help() -> None:
     [dark_green]archivebox[/dark_green] [green]add[/green] --depth=1 "https://example.com/some/page"
     [dark_green]archivebox[/dark_green] [green]list[/green] --sort=timestamp --csv=timestamp,downloaded_at,url,title
     [dark_green]archivebox[/dark_green] [green]schedule[/green] --every=day --depth=1 "https://example.com/some/feed.rss"
-    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:5797[/blue]                [grey53]# Start the Web UI / API server[/grey53]
+    [dark_green]archivebox[/dark_green] [green]server[/green] [blue]0.0.0.0:8000[/blue]                [grey53]# Start the Web UI / API server[/grey53]
 """
         print(
             Panel(

archivebox/cli/archivebox_init.py

@@ -25,20 +25,20 @@ def _display_data_path(path: Path, data_dir: Path) -> str:
 def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
     """Initialize a new ArchiveBox collection in the current directory"""
 
-    from archivebox.misc.checks import check_not_inside_source_dir
-
-    check_not_inside_source_dir()
-
-    from archivebox.config import CONSTANTS, VERSION
+    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
     from archivebox.config.common import get_config
     from archivebox.config.collection import write_config_file
-    from archivebox.misc.db import apply_migrations, database_exists, ensure_database_ready
+    from archivebox.misc.db import apply_migrations
     from archivebox.misc.checks import check_migrations
 
     config = get_config()
 
-    is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
-    existing_index = database_exists()
+    # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
+    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
+    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
+
+    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
+    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
     if is_empty and not existing_index:
         print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
         print("[green]----------------------------------------------------------------------[/green]")
@@ -65,70 +65,44 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     else:
         print("\n[green][+] Building archive folder structure...[/green]")
 
-    archive_path = _display_data_path(CONSTANTS.ARCHIVE_DIR, CONSTANTS.DATA_DIR)
-    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, CONSTANTS.DATA_DIR)
-    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, CONSTANTS.DATA_DIR)
+    archive_path = _display_data_path(config.ARCHIVE_DIR, DATA_DIR)
+    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, DATA_DIR)
+    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, DATA_DIR)
     print(f"    + {archive_path}, {sources_path}, {logs_path}...")
     Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
-    CONSTANTS.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
-    CONSTANTS.USERS_DIR.mkdir(parents=True, exist_ok=True)
+    config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
+    config.USERS_DIR.mkdir(parents=True, exist_ok=True)
     Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
-    for path in (Path(CONSTANTS.SOURCES_DIR), CONSTANTS.ARCHIVE_DIR, CONSTANTS.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
-        path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)
 
-    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, CONSTANTS.DATA_DIR)}...")
+    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...")
 
     # create the .archivebox_id file with a unique ID for this collection
     from archivebox.config.paths import _get_collection_id
 
-    _get_collection_id(CONSTANTS.DATA_DIR, force_create=True)
+    _get_collection_id(DATA_DIR, force_create=True)
 
     # create the ArchiveBox.conf file
     write_config_file({"SECRET_KEY": config.SECRET_KEY})
 
-    if existing_index:
+    if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
         print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]")
     else:
         print("\n[green][+] Building main SQL index and running initial migrations...[/green]")
 
-    # For postgres, make sure the server is reachable and create the database
-    # if it doesn't exist yet (sqlite creates its file automatically).
-    ensure_database_ready()
-
     from archivebox.config.django import setup_django
 
     setup_django()
-    previous_wants_init = os.environ.get("ARCHIVEBOX_WANTS_INIT")
-    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
-    try:
-        check_migrations(blocking=True, auto_apply=False)
+    check_migrations(blocking=True, auto_apply=False)
 
-        for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
-            sys.stdout.write(f"    {migration_line}\n")
-    finally:
-        if previous_wants_init is None:
-            os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
-        else:
-            os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
+    for migration_line in apply_migrations(DATA_DIR):
+        sys.stdout.write(f"    {migration_line}\n")
 
-    from archivebox.machine.models import Machine
-
-    # Registration requires the final schema and reconciles legacy file config.
-    Machine.current()
-
-    from archivebox.personas.models import Persona
-
-    # Initialize here, not when /add/ is rendered: a GET must not create profiles.
-    Persona.get_or_create_default().ensure_dirs()
-
-    from archivebox.misc.db import database_display_location, is_postgres
-
-    assert database_exists()
+    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
     print()
-    if is_postgres():
-        print(f"    √ {database_display_location()}")
-    else:
-        print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
+    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, DATA_DIR)}")
+
+    # from django.contrib.auth.models import User
+    #     call_command("createsuperuser", interactive=True)
 
     print()
     print("[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]")
@@ -144,7 +118,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     print("    > Skipping orphan snapshot import during init.")
     print()
     print("    [violet]Hint:[/violet] To import orphaned snapshot directories and reconcile filesystem state, run:")
-    print("        archivebox update --rescan --migrate-only")
+    print("        archivebox update")
 
     print("\n[green]----------------------------------------------------------------------[/green]")
 
@@ -165,12 +139,12 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
     CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
 
-    from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir
+    from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir
 
     config = get_config()
-    create_and_chown_dir(config.TMP_DIR)
-    create_and_chown_dir(config.ABXPKG_LIB_DIR)
-    create_and_chown_dir(config.ABXPKG_LIB_DIR / "bin")
+    config.TMP_DIR.mkdir(parents=True, exist_ok=True)
+    config.LIB_DIR.mkdir(parents=True, exist_ok=True)
+    (config.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
 
     working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
     if working_tmp_dir:
@@ -179,6 +153,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
     if working_lib_dir:
         working_lib_dir.mkdir(parents=True, exist_ok=True)
+        (working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)
 
     if install:
         from archivebox.cli.archivebox_install import install as install_method
@@ -186,13 +161,10 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
         install_method()
 
     if Snapshot.objects.count() < 25:  # hide the hints for experienced users
-        from archivebox.core.routes_util import build_admin_url
-
-        admin_url = build_admin_url("/admin/", config=config)
         print()
-        print("    [violet]Hint:[/violet] To finish web setup, run:")
+        print("    [violet]Hint:[/violet] To view your archive index, run:")
         print(
-            f"        archivebox server  # then visit [deep_sky_blue4][link={admin_url}]{admin_url}[/link][/deep_sky_blue4]",
+            "        archivebox server  # then visit [deep_sky_blue4][link=http://127.0.0.1:8000]http://127.0.0.1:8000[/link][/deep_sky_blue4]",
         )
         print()
         print("    To add new links, you can run:")
@@ -205,7 +177,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
 @click.command()
 @click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
 @click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
-@click.option("--install", "--setup", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
+@click.option("--install", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
 @docstring(init.__doc__)
 def main(**kwargs) -> None:
     init(**kwargs)

archivebox/cli/archivebox_install.py

@@ -3,9 +3,6 @@
 __package__ = "archivebox.cli"
 
 import os
-import shutil
-import re
-from pathlib import Path
 
 import rich_click as click
 from rich import print
@@ -13,113 +10,6 @@ from rich import print
 from archivebox.misc.util import docstring, enforce_types
 
 
-def _resolve_install_targets(
-    requested_names: tuple[str, ...],
-) -> tuple[list[str], list[str]]:
-    """Resolve plugin names and declared binary aliases, leaving unknown tokens raw."""
-    from archivebox.plugins.discovery import get_plugin_catalog
-
-    catalog = get_plugin_catalog()
-    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in catalog}
-    plugin_names_by_binary_alias: dict[str, set[str]] = {}
-    for plugin_name, plugin in catalog.items():
-        for required_binary in plugin.config.required_binaries:
-            aliases = {required_binary.name.lower()}
-            template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", required_binary.name)
-            if template_match:
-                config_key = template_match.group(1)
-                aliases.add(config_key.lower())
-                aliases.add(config_key.removesuffix("_BINARY").lower())
-                default = plugin.config.properties.get(config_key, {}).get("default")
-                if default:
-                    aliases.add(str(default).lower())
-            for alias in aliases:
-                plugin_names_by_binary_alias.setdefault(alias, set()).add(plugin_name)
-
-    requested_plugins: list[str] = []
-    raw_binary_names: list[str] = []
-    for name in requested_names:
-        normalized_name = name.lower()
-        if normalized_name in plugin_names_by_lower:
-            requested_plugins.append(plugin_names_by_lower[normalized_name])
-        elif normalized_name in plugin_names_by_binary_alias:
-            requested_plugins.extend(sorted(plugin_names_by_binary_alias[normalized_name]))
-        else:
-            raw_binary_names.append(name)
-
-    selected_plugins = catalog.select(requested_plugins) if requested_plugins else catalog.select([])
-    selected_plugin_names = {name.lower() for name in selected_plugins}
-    raw_binary_names = [name for name in raw_binary_names if name.lower() not in selected_plugin_names]
-    return sorted(selected_plugins), sorted(set(raw_binary_names))
-
-
-def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> None:
-    """Install user-requested standalone binaries through the Binary lifecycle."""
-    from django.utils import timezone
-
-    from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-
-    machine = Machine.current()
-    for requested_name in binary_names:
-        binary_name = _canonical_binary_name(requested_name)
-        if not binary_name:
-            raise click.UsageError(f"Invalid binary name: {requested_name}")
-        binary = Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").first()
-        if binary is None:
-            binary = Binary.objects.create(
-                machine=machine,
-                name=binary_name,
-                binproviders=binproviders,
-                overrides={},
-                status=Binary.StatusChoices.QUEUED,
-            )
-        elif not binary.is_valid or binary.binproviders != binproviders:
-            binary.binproviders = binproviders
-            binary.overrides = {}
-            binary.status = Binary.StatusChoices.QUEUED
-            binary.retry_at = timezone.now()
-            binary.save(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
-        if binary.status == Binary.StatusChoices.INSTALLED:
-            continue
-        # The synchronous CLI needs the lifecycle's verified installation result,
-        # not the scheduler's "claimed work" result. Reuse its failure/requeue and
-        # health-stat handling so a failed install cannot exit successfully.
-        if not binary.install_claimed(lock_seconds=60):
-            raise click.ClickException(f"Binary {binary_name} is already being installed")
-
-
-def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:
-    """Expose the shared abxpkg lib dir at DATA_DIR/lib after install."""
-    link_path = data_dir / "lib"
-    target_config_path = abxpkg_lib_dir.expanduser().absolute()
-    target_path = abxpkg_lib_dir.expanduser().resolve()
-    from archivebox.config.paths import get_machine_type
-
-    scoped_data_lib_path = (link_path / get_machine_type()).absolute()
-
-    if target_config_path == scoped_data_lib_path:
-        if link_path.is_symlink() or link_path.is_file():
-            link_path.unlink()
-        target_config_path.mkdir(parents=True, exist_ok=True)
-        return None
-
-    if link_path.resolve(strict=False) == target_path:
-        return None
-
-    target_path.mkdir(parents=True, exist_ok=True)
-
-    if link_path.is_symlink():
-        if link_path.resolve(strict=False) == target_path:
-            return link_path
-        link_path.unlink()
-    elif link_path.exists():
-        shutil.rmtree(link_path)
-
-    relative_target = os.path.relpath(target_path, start=link_path.parent)
-    link_path.symlink_to(relative_target, target_is_directory=True)
-    return link_path
-
-
 @enforce_types
 def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
     """Detect and install ArchiveBox dependencies by running the abx-dl install flow
@@ -132,16 +22,12 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
     """
 
     from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
-    from archivebox.config import CONSTANTS
+    from archivebox.config.common import get_config
     from archivebox.misc.logging import stderr
     from archivebox.cli.archivebox_init import init
 
-    archive_dir = CONSTANTS.ARCHIVE_DIR
-
-    if dry_run:
-        print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")
-        return
-
+    config = get_config()
+    archive_dir = config.ARCHIVE_DIR
     if not (os.access(archive_dir, os.R_OK) and archive_dir.is_dir()):
         init()  # must init full index because we need a db to store Binary entries in
 
@@ -161,37 +47,25 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
         print(f"    DATA_DIR will be owned by [blue]{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}[/blue].")
         print()
 
+    if dry_run:
+        print("[dim]Dry run - would run the abx-dl install flow[/dim]")
+        return
+
     # Set up Django
     from archivebox.config.django import setup_django
 
     setup_django()
 
-    plugin_names, raw_binary_names = _resolve_install_targets(binaries) if binaries else ([], [])
-    install_plugin_names = list(plugin_names)
+    plugin_names = list(binaries)
     if binproviders != "*":
-        install_plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
+        plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
 
-    if install_plugin_names or not binaries:
-        if not binaries:
-            from archivebox.workers.supervisord_util import require_dependency_install_memory
+    print("[+] Running installer via abx-dl bus...")
+    print()
 
-            require_dependency_install_memory()
-        print("[+] Running plugin installer via abx-dl bus...")
-        print()
+    from archivebox.services.runner import run_install
 
-        from archivebox.services.runner import run_install
-
-        run_install(plugin_names=install_plugin_names or None)
-
-    if raw_binary_names:
-        print(f"[+] Running direct binary installer via ArchiveBox binary lifecycle: {', '.join(raw_binary_names)}")
-        print()
-        _install_raw_binary_names(raw_binary_names, binproviders)
-
-    from archivebox.config.common import get_config
-
-    config = get_config(include_machine=False)
-    ensure_data_dir_lib_symlink(CONSTANTS.DATA_DIR, config.ABXPKG_LIB_DIR)
+    run_install(plugin_names=plugin_names or None)
 
     print()
 
@@ -201,10 +75,8 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
     User = get_user_model()
 
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
-        from archivebox.core.routes_util import build_admin_url
-
-        stderr("\n[+] Open the Admin UI to create the first admin and finish web setup:", color="green")
-        stderr(f"    {build_admin_url('/admin/', config=get_config(resolve_plugins=False))}")
+        stderr("\n[+] Don't forget to create a new admin user for the Web UI...", color="green")
+        stderr("    archivebox manage createsuperuser")
 
     print()
 
@@ -212,7 +84,7 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
     # Django is already loaded, so just import and call the function directly
     from archivebox.cli.archivebox_version import version as show_version
 
-    show_version(quiet=False, binaries=binaries)
+    show_version(quiet=False)
 
 
 @click.command()

archivebox/cli/archivebox_list.py

@@ -7,15 +7,50 @@ import sys
 
 import rich_click as click
 
-from archivebox.cli.archivebox_snapshot import list_snapshots, snapshot_filter_options, snapshot_output_options
+from archivebox.cli.archivebox_snapshot import list_snapshots
 
 
 @click.command()
-@snapshot_output_options
-@snapshot_filter_options(default_filter_type="substring")
-def main(**kwargs) -> None:
+@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
+@click.option("--url__icontains", help="Filter by URL contains")
+@click.option("--url__istartswith", help="Filter by URL starts with")
+@click.option("--tag", "-t", help="Filter by tag name")
+@click.option("--crawl-id", help="Filter by crawl ID")
+@click.option("--limit", "-n", type=int, help="Limit number of results")
+@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
+@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
+@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
+@click.option("--search", type=click.Choice(["meta", "content", "contents", "deep"]), help="Search mode to use for the query")
+@click.argument("query", nargs=-1)
+def main(
+    status: str | None,
+    url__icontains: str | None,
+    url__istartswith: str | None,
+    tag: str | None,
+    crawl_id: str | None,
+    limit: int | None,
+    sort: str | None,
+    csv: str | None,
+    with_headers: bool,
+    search: str | None,
+    query: tuple[str, ...],
+) -> None:
     """List Snapshots."""
-    sys.exit(list_snapshots(**kwargs))
+    sys.exit(
+        list_snapshots(
+            status=status,
+            url__icontains=url__icontains,
+            url__istartswith=url__istartswith,
+            tag=tag,
+            crawl_id=crawl_id,
+            limit=limit,
+            sort=sort,
+            csv=csv,
+            with_headers=with_headers,
+            search=search,
+            query=" ".join(query),
+        ),
+    )
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_machine.py

@@ -27,7 +27,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_util import apply_filters
+from archivebox.cli.cli_utils import apply_filters
 
 
 # =============================================================================

archivebox/cli/archivebox_manage.py

@@ -14,7 +14,7 @@ def manage(args: list[str] | None = None) -> None:
     from archivebox.misc.logging import stderr
 
     config = get_config()
-    if (args and "createsuperuser" in args and "--noinput" not in args) and (config.IN_DOCKER and not config.IS_TTY):
+    if (args and "createsuperuser" in args) and (config.IN_DOCKER and not config.IS_TTY):
         stderr("[!] Warning: you need to pass -it to use interactive commands in docker", color="lightyellow")
         stderr("    docker run -it archivebox manage {}".format(" ".join(args or ["..."])), color="lightyellow")
         stderr("")

archivebox/cli/archivebox_oneshot.py

@@ -1,49 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox oneshot"
-
-import subprocess
-import sys
-from pathlib import Path
-
-import rich_click as click
-
-from archivebox.config import CONSTANTS
-from archivebox.config.common import get_config
-
-
-@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
-@click.argument("args", nargs=-1)
-def main(args: tuple[str, ...] = ()) -> None:
-    """Download URLs using abx-dl"""
-    from archivebox.misc.checks import check_not_inside_source_dir
-
-    check_not_inside_source_dir()
-
-    cwd = Path.cwd()
-    if any((path / CONSTANTS.SQL_INDEX_FILENAME).exists() for path in (cwd, *cwd.parents)):
-        raise click.ClickException(
-            "Refusing to run `archivebox oneshot` inside an ArchiveBox DATA_DIR. Use `archivebox add` here, or run oneshot from another directory.",
-        )
-    abxpkg_binary = Path(sys.executable).with_name("abxpkg")
-    if not abxpkg_binary.is_file():
-        raise click.ClickException(f"abxpkg executable is missing from the ArchiveBox environment: {abxpkg_binary}")
-    abxpkg_lib_dir = get_config(include_machine=False).ABXPKG_LIB_DIR
-    raise SystemExit(
-        subprocess.run(
-            [
-                str(abxpkg_binary),
-                f"--lib={abxpkg_lib_dir}",
-                "--binproviders=env",
-                "--install",
-                "run",
-                "abx-dl",
-                *args,
-            ],
-        ).returncode,
-    )
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_persona.py

@@ -27,19 +27,189 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox persona"
 
+import os
 import sys
 import shutil
+import platform
 from pathlib import Path
-from dataclasses import replace
 from collections.abc import Iterable
 
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_util import apply_filters
+from archivebox.cli.cli_utils import apply_filters
 from archivebox.personas import importers as persona_importers
 
 
+# =============================================================================
+# Browser Profile Locations
+# =============================================================================
+
+
+def get_chrome_user_data_dir() -> Path | None:
+    """Get the default Chrome user data directory for the current platform."""
+    system = platform.system()
+    home = Path.home()
+
+    if system == "Darwin":  # macOS
+        candidates = [
+            home / "Library" / "Application Support" / "Google" / "Chrome",
+            home / "Library" / "Application Support" / "Chromium",
+        ]
+    elif system == "Linux":
+        candidates = [
+            home / ".config" / "google-chrome",
+            home / ".config" / "chromium",
+            home / ".config" / "chrome",
+            home / "snap" / "chromium" / "common" / "chromium",
+        ]
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = [
+            local_app_data / "Google" / "Chrome" / "User Data",
+            local_app_data / "Chromium" / "User Data",
+        ]
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate.exists() and (candidate / "Default").exists():
+            return candidate
+
+    return None
+
+
+def get_brave_user_data_dir() -> Path | None:
+    """Get the default Brave user data directory for the current platform."""
+    system = platform.system()
+    home = Path.home()
+
+    if system == "Darwin":
+        candidates = [
+            home / "Library" / "Application Support" / "BraveSoftware" / "Brave-Browser",
+        ]
+    elif system == "Linux":
+        candidates = [
+            home / ".config" / "BraveSoftware" / "Brave-Browser",
+        ]
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = [
+            local_app_data / "BraveSoftware" / "Brave-Browser" / "User Data",
+        ]
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate.exists() and (candidate / "Default").exists():
+            return candidate
+
+    return None
+
+
+def get_edge_user_data_dir() -> Path | None:
+    """Get the default Edge user data directory for the current platform."""
+    system = platform.system()
+    home = Path.home()
+
+    if system == "Darwin":
+        candidates = [
+            home / "Library" / "Application Support" / "Microsoft Edge",
+        ]
+    elif system == "Linux":
+        candidates = [
+            home / ".config" / "microsoft-edge",
+            home / ".config" / "microsoft-edge-beta",
+            home / ".config" / "microsoft-edge-dev",
+        ]
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = [
+            local_app_data / "Microsoft" / "Edge" / "User Data",
+        ]
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate.exists() and (candidate / "Default").exists():
+            return candidate
+
+    return None
+
+
+def get_browser_binary(browser: str) -> str | None:
+    system = platform.system()
+    home = Path.home()
+    browser = browser.lower()
+
+    if system == "Darwin":
+        candidates = {
+            "chrome": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"],
+            "chromium": ["/Applications/Chromium.app/Contents/MacOS/Chromium"],
+            "brave": ["/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"],
+            "edge": ["/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"],
+        }.get(browser, [])
+    elif system == "Linux":
+        candidates = {
+            "chrome": [
+                "/usr/bin/google-chrome",
+                "/usr/bin/google-chrome-stable",
+                "/usr/bin/google-chrome-beta",
+                "/usr/bin/google-chrome-unstable",
+            ],
+            "chromium": ["/usr/bin/chromium", "/usr/bin/chromium-browser"],
+            "brave": ["/usr/bin/brave-browser", "/usr/bin/brave-browser-beta", "/usr/bin/brave-browser-nightly"],
+            "edge": [
+                "/usr/bin/microsoft-edge",
+                "/usr/bin/microsoft-edge-stable",
+                "/usr/bin/microsoft-edge-beta",
+                "/usr/bin/microsoft-edge-dev",
+            ],
+        }.get(browser, [])
+    elif system == "Windows":
+        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
+        candidates = {
+            "chrome": [
+                str(local_app_data / "Google" / "Chrome" / "Application" / "chrome.exe"),
+                "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
+                "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
+            ],
+            "chromium": [str(local_app_data / "Chromium" / "Application" / "chrome.exe")],
+            "brave": [
+                str(local_app_data / "BraveSoftware" / "Brave-Browser" / "Application" / "brave.exe"),
+                "C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
+                "C:\\Program Files (x86)\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
+            ],
+            "edge": [
+                str(local_app_data / "Microsoft" / "Edge" / "Application" / "msedge.exe"),
+                "C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
+                "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
+            ],
+        }.get(browser, [])
+    else:
+        candidates = []
+
+    for candidate in candidates:
+        if candidate and Path(candidate).exists():
+            return candidate
+
+    return None
+
+
+BROWSER_PROFILE_FINDERS = {
+    "chrome": get_chrome_user_data_dir,
+    "chromium": get_chrome_user_data_dir,  # Same locations
+    "brave": get_brave_user_data_dir,
+    "edge": get_edge_user_data_dir,
+}
+
+CHROMIUM_BROWSERS = {"chrome", "chromium", "brave", "edge"}
+
+
+# =============================================================================
+# Cookie Extraction via CDP
+# =============================================================================
+
 # =============================================================================
 # Validation Helpers
 # =============================================================================
@@ -107,9 +277,6 @@ def create_personas(
     names: Iterable[str],
     import_from: str | None = None,
     profile: str | None = None,
-    source: str | None = None,
-    browser_binary: str | None = None,
-    permissions: str | None = None,
 ) -> int:
     """
     Create Personas from names.
@@ -131,26 +298,29 @@ def create_personas(
         rprint("[yellow]No persona names provided. Pass names as arguments.[/yellow]", file=sys.stderr)
         return 1
 
-    import_source = None
-    if source and not import_from:
-        rprint("[red]--source requires --import (the source browser name).[/red]", file=sys.stderr)
-        return 1
+    # Validate import source if specified
+    source_profile_dir = None
     if import_from:
-        try:
-            if source:
-                import_source = persona_importers.resolve_custom_import_source(source, profile_dir=profile)
-                if import_source.browser != "persona":
-                    import_source = replace(import_source, browser=import_from.lower(), browser_binary=browser_binary)
-            elif import_from.startswith(("http://", "https://", "ws://", "wss://")) or Path(import_from).expanduser().is_absolute():
-                import_source = persona_importers.resolve_custom_import_source(import_from)
-            else:
-                import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile)
-            if browser_binary:
-                import_source = replace(import_source, browser_binary=browser_binary)
-        except ValueError as err:
-            rprint(f"[red]{err}[/red]", file=sys.stderr)
+        import_from = import_from.lower()
+        if import_from not in BROWSER_PROFILE_FINDERS:
+            rprint(f"[red]Unknown browser: {import_from}[/red]", file=sys.stderr)
+            rprint(f"[dim]Supported browsers: {', '.join(BROWSER_PROFILE_FINDERS.keys())}[/dim]", file=sys.stderr)
             return 1
 
+        source_profile_dir = BROWSER_PROFILE_FINDERS[import_from]()
+        if not source_profile_dir:
+            rprint(f"[red]Could not find {import_from} profile directory[/red]", file=sys.stderr)
+            return 1
+
+        rprint(f"[dim]Found {import_from} profile: {source_profile_dir}[/dim]", file=sys.stderr)
+
+        if profile is None and (source_profile_dir / "Default").exists():
+            profile = "Default"
+
+        browser_binary = get_browser_binary(import_from)
+        if browser_binary:
+            rprint(f"[dim]Using {import_from} binary: {browser_binary}[/dim]", file=sys.stderr)
+
     created_count = 0
     for name in name_list:
         name = name.strip()
@@ -163,25 +333,21 @@ def create_personas(
             rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
             continue
 
-        created = Persona.find_named(name) is None
-        persona = Persona.get_or_create_named(
-            name,
-            defaults={"config": {"PERMISSIONS": permissions}} if permissions else {},
-        )
+        persona, created = Persona.objects.get_or_create(name=name)
 
         if created:
             persona.ensure_dirs()
             created_count += 1
-
+            rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
         else:
             rprint(f"[dim]Persona already exists: {name}[/dim]", file=sys.stderr)
 
         cookies_file = Path(persona.path) / "cookies.txt"
 
         # Import browser profile if requested
-        if import_source is not None:
+        if import_from in CHROMIUM_BROWSERS and source_profile_dir is not None:
             try:
-                rprint(f"[dim]Importing {import_source.display_label}; copying settings and exporting cookies...[/dim]", file=sys.stderr)
+                import_source = persona_importers.resolve_browser_import_source(import_from, profile_dir=profile)
                 import_result = persona_importers.import_persona_from_source(
                     persona,
                     import_source,
@@ -189,29 +355,24 @@ def create_personas(
                     import_cookies=True,
                     capture_storage=False,
                 )
-            except (Exception, KeyboardInterrupt) as e:
-                if created:
-                    shutil.rmtree(persona.path, ignore_errors=True)
-                    persona.delete()
+            except Exception as e:
                 rprint(f"[red]Failed to import browser profile: {e}[/red]", file=sys.stderr)
                 return 1
 
             if import_result.profile_copied:
                 rprint("[green]Copied browser profile to persona[/green]", file=sys.stderr)
             if import_result.cookies_imported:
-                rprint(f"[green]Extracted {import_result.cookie_count} cookies to {cookies_file}[/green]", file=sys.stderr)
+                rprint(f"[green]Extracted cookies to {cookies_file}[/green]", file=sys.stderr)
             elif not import_result.profile_copied:
                 rprint("[yellow]Could not import cookies automatically.[/yellow]", file=sys.stderr)
 
             for warning in import_result.warnings:
                 rprint(f"[yellow]{warning}[/yellow]", file=sys.stderr)
 
-        if created:
-            rprint(f"[green]Created persona: {name}[/green]", file=sys.stderr)
         if not is_tty:
             write_record(
                 {
-                    "id": str(persona.id),
+                    "id": str(persona.id) if hasattr(persona, "id") else None,
                     "name": persona.name,
                     "path": str(persona.path),
                     "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -263,7 +424,7 @@ def list_personas(
         else:
             write_record(
                 {
-                    "id": str(persona.id),
+                    "id": str(persona.id) if hasattr(persona, "id") else None,
                     "name": persona.name,
                     "path": str(persona.path),
                     "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -314,9 +475,7 @@ def update_personas(name: str | None = None) -> int:
             if persona_id:
                 persona = Persona.objects.get(id=persona_id)
             else:
-                persona = Persona.find_named(old_name)
-                if persona is None:
-                    raise Persona.DoesNotExist
+                persona = Persona.objects.get(name=old_name)
 
             # Apply updates from CLI flags
             if name:
@@ -328,7 +487,7 @@ def update_personas(name: str | None = None) -> int:
 
                 # Rename the persona directory too
                 old_path = persona.path
-                persona.name = persona.validate_name(name)
+                persona.name = name
                 new_path = persona.path
 
                 if old_path.exists() and old_path != new_path:
@@ -341,7 +500,7 @@ def update_personas(name: str | None = None) -> int:
             if not is_tty:
                 write_record(
                     {
-                        "id": str(persona.id),
+                        "id": str(persona.id) if hasattr(persona, "id") else None,
                         "name": persona.name,
                         "path": str(persona.path),
                     },
@@ -448,42 +607,11 @@ def main():
 
 @main.command("create")
 @click.argument("names", nargs=-1)
-@click.option(
-    "--import",
-    "import_from",
-    help="Import from chrome, chrome-beta, chromium, brave, edge, a persona directory, or a live CDP URL",
-)
-@click.option("--source", help="Source browser user-data directory or exact profile path (including Docker mounts)")
-@click.option(
-    "--browser-binary",
-    type=click.Path(exists=True, dir_okay=False),
-    help="Source Chromium browser executable; required for other Chromium-based browsers",
-)
+@click.option("--import", "import_from", help="Import profile from browser (chrome, chromium, brave, edge)")
 @click.option("--profile", help="Profile directory name under the user data dir (e.g. Default, Profile 1)")
-@click.option(
-    "--permissions",
-    type=click.Choice(["public", "unlisted", "private"]),
-    help="Visibility for a newly created persona (existing personas are unchanged)",
-)
-def create_cmd(
-    names: tuple,
-    import_from: str | None,
-    profile: str | None,
-    source: str | None,
-    browser_binary: str | None,
-    permissions: str | None,
-):
+def create_cmd(names: tuple, import_from: str | None, profile: str | None):
     """Create Personas, optionally importing from a browser profile."""
-    sys.exit(
-        create_personas(
-            names,
-            import_from=import_from,
-            profile=profile,
-            source=source,
-            browser_binary=browser_binary,
-            permissions=permissions,
-        ),
-    )
+    sys.exit(create_personas(names, import_from=import_from, profile=profile))
 
 
 @main.command("list")
@@ -495,84 +623,6 @@ def list_cmd(name: str | None, name__icontains: str | None, limit: int | None):
     sys.exit(list_personas(name=name, name__icontains=name__icontains, limit=limit))
 
 
-@main.command("open")
-@click.argument("name", default="Default")
-def open_cmd(name: str):
-    """Open a persona's browser, including its imported logins."""
-    import json
-    import os
-    import subprocess
-
-    import psutil
-
-    from abx_plugins import get_plugins_dir
-    from archivebox.config.common import get_config
-    from archivebox.personas.models import Persona
-
-    valid, reason = validate_persona_name(name)
-    if not valid:
-        raise click.ClickException(reason)
-    persona = Persona.get_or_create_named(name)
-    persona.ensure_dirs()
-    config = get_config(persona=persona)
-    if os.environ.get("DISPLAY"):
-        from archivebox.config import CONSTANTS
-
-        for process in psutil.process_iter():
-            try:
-                args = process.cmdline()
-                profile = next((arg.split("=", 1)[1] for arg in args if arg.startswith("--user-data-dir=")), None)
-                if (
-                    not profile
-                    or any(arg.startswith(("--type=", "--headless")) for arg in args)
-                    or not Path(profile).resolve().is_relative_to(CONSTANTS.DATA_DIR.resolve())
-                    or process.environ().get("DISPLAY") != os.environ["DISPLAY"]
-                ):
-                    continue
-                rprint("An ArchiveBox browser is already open on this display; waiting for it to close.")
-                process.wait()
-            except (psutil.NoSuchProcess, psutil.AccessDenied):
-                continue
-    plugins_dir = Path(get_plugins_dir()).resolve()
-    env = os.environ.copy()
-    dependencies = subprocess.run(
-        [
-            str(Path(sys.executable).with_name("abxpkg")),
-            "env",
-            "--install",
-            "--json",
-            f"--lib={config.ABXPKG_LIB_DIR}",
-            f"--deps-from={plugins_dir / 'chrome' / 'config.json'}:required_binaries",
-        ],
-        capture_output=True,
-        text=True,
-        env=env,
-    )
-    if dependencies.returncode:
-        raise click.ClickException(dependencies.stderr.strip())
-    env.update({key: str(value) for key, value in json.loads(dependencies.stdout).items()})
-    env["ARCHIVEBOX_ABX_PLUGINS_DIR"] = str(plugins_dir)
-    env["ACTIVE_PERSONA"] = persona.name
-    env["PERSONAS_DIR"] = str(persona.path.parent)
-    env["NODE_MODULES_DIR"] = str(config.ABXPKG_LIB_DIR / "pnpm" / "packages" / "chrome" / "node_modules")
-    payload = config.model_dump(mode="json")
-    payload.update(
-        ACTIVE_PERSONA=persona.name,
-        CHROME_HEADLESS=False,
-        CHROME_USER_DATA_DIR=str(persona.CHROME_USER_DATA_DIR),
-        AUTH_STORAGE_FILE=str(persona.AUTH_STORAGE_FILE or ""),
-        COOKIES_FILE=str(persona.COOKIES_FILE or ""),
-    )
-    script = Path(__file__).parent.parent / "personas" / "open_browser.js"
-    result = subprocess.run(
-        [env.get("NODE_BINARY") or "node", str(script)],
-        input=json.dumps(payload),
-        text=True,
-        env=env,
-    )
-    sys.exit(result.returncode)
-
-
 @main.command("update")
 @click.option("--name", "-n", help="Set new name")
 def update_cmd(name: str | None):

archivebox/cli/archivebox_pluginmap.py

@@ -16,9 +16,8 @@ EVENT_FLOW_DIAGRAM = """
 │  InstallEvent                                                               │
 │    └─ config.json > required_binaries                                       │
 │         └─ BinaryRequestEvent                                               │
-│              └─ abxpkg BinaryService builtin providers                      │
+│              └─ on_BinaryRequest__*                                         │
 │                   └─ BinaryEvent                                            │
-│                        └─ ArchiveBox DB history projection                  │
 │                                                                             │
 │  CrawlEvent                                                                 │
 │    └─ CrawlSetupEvent                                                       │
@@ -48,7 +47,7 @@ def pluginmap(
     Show the current abx-dl event phases and their associated plugin hooks.
 
     This command reflects the new bus-driven runtime, not the legacy ArchiveBox
-    event runtime. Event names are normalized to hook prefixes by
+    state-machine executor. Event names are normalized to hook prefixes by
     stripping a trailing `Event`, then ArchiveBox checks whether any matching
     `on_{EventFamily}__*` scripts actually exist.
     """
@@ -57,15 +56,13 @@ def pluginmap(
     from rich.panel import Panel
     from rich import box
 
-    from archivebox.plugins.hooks import (
+    from archivebox.hooks import (
+        BUILTIN_PLUGINS_DIR,
+        USER_PLUGINS_DIR,
         discover_hooks,
         is_background_hook,
         normalize_hook_event_name,
     )
-    from archivebox.plugins.discovery import (
-        BUILTIN_PLUGINS_DIR,
-        USER_PLUGINS_DIR,
-    )
 
     console = Console()
     prnt = console.print
@@ -76,14 +73,12 @@ def pluginmap(
             "emits": ["BinaryRequestEvent", "BinaryEvent", "ProcessEvent"],
         },
         "BinaryRequestEvent": {
-            "description": "Binary resolution phase. abxpkg BinaryService resolves or installs requested binaries using built-in providers.",
+            "description": "Provider phase. on_BinaryRequest hooks resolve or install requested binaries.",
             "emits": ["BinaryEvent", "ProcessEvent"],
-            "direct_hooks": False,
         },
         "BinaryEvent": {
-            "description": "Resolved binary metadata event. ArchiveBoxBinaryService projects it into the ArchiveBox DB binary cache.",
+            "description": "Resolved binary metadata event. Projected into the DB binary cache.",
             "emits": [],
-            "direct_hooks": False,
         },
         "CrawlEvent": {
             "description": "Root crawl lifecycle event emitted by the runner.",
@@ -143,8 +138,7 @@ def pluginmap(
 
     for event_name, info in event_phases.items():
         hook_event = normalize_hook_event_name(event_name)
-        has_direct_hooks = info.get("direct_hooks", True)
-        hooks = discover_hooks(event_name, filter_disabled=not show_disabled) if has_direct_hooks else []
+        hooks = discover_hooks(event_name, filter_disabled=not show_disabled)
 
         hook_infos = []
         for hook_path in hooks:
@@ -198,10 +192,8 @@ def pluginmap(
         prnt(f"[dim]{info['description']}[/dim]")
         if info["emits"]:
             prnt(f"[dim]Emits: {', '.join(info['emits'])}[/dim]")
-        if not hook_infos and has_direct_hooks:
+        if not hook_infos:
             prnt(f"[dim]No direct on_{hook_event}__* scripts are currently defined for this event family.[/dim]")
-        elif not has_direct_hooks:
-            prnt("[dim]No direct plugin hook family. This event is handled by services.[/dim]")
         prnt()
 
     if not quiet:

archivebox/cli/archivebox_process.py

@@ -30,7 +30,7 @@ import sys
 import rich_click as click
 from rich import print as rprint
 
-from archivebox.cli.cli_util import apply_filters
+from archivebox.cli.cli_utils import apply_filters
 
 
 # =============================================================================

archivebox/cli/archivebox_remove.py

@@ -3,7 +3,6 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox remove"
 
-import time
 from pathlib import Path
 from collections.abc import Iterable
 
@@ -11,7 +10,7 @@ import rich_click as click
 
 from django.db.models import QuerySet
 
-from archivebox.config import CONSTANTS
+from archivebox.config import DATA_DIR
 from archivebox.config.django import setup_django
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.checks import check_data_folder
@@ -22,7 +21,6 @@ from archivebox.misc.logging_util import (
     log_removal_finished,
     TimedProgress,
 )
-from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
 
 @enforce_types
@@ -33,103 +31,66 @@ def remove(
     after: float | None = None,
     before: float | None = None,
     yes: bool = False,
-    delete: bool = False,
-    out_dir: Path = CONSTANTS.DATA_DIR,
-    timeout: float | None = None,
-    **kwargs,
-) -> dict[str, object]:
+    out_dir: Path = DATA_DIR,
+) -> QuerySet:
     """Remove the specified URLs from the archive"""
 
     setup_django()
     check_data_folder()
-    timeout = float(timeout) if timeout is not None else None
 
-    from archivebox.core.models import Snapshot
+    from archivebox.cli.archivebox_search import get_snapshots
 
-    filter_kwargs = {
-        **kwargs,
-        "filter_patterns": filter_patterns,
-        "filter_type": filter_type,
-        "after": after,
-        "before": before,
-    }
     pattern_list = list(filter_patterns)
 
     log_list_started(pattern_list or None, filter_type)
     timer = TimedProgress(360, prefix="      ")
     try:
-        if snapshots is None:
-            snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
-        # Freeze the target set up-front so a concurrent daemon writing new
-        # snapshots can't extend the deletion under us, and so the cursor isn't
-        # held open across the per-row deletes below.
-        snapshot_pks = list(snapshots.values_list("pk", flat=True))
+        snapshots = get_snapshots(
+            snapshots=snapshots,
+            filter_patterns=pattern_list or None,
+            filter_type=filter_type,
+            after=after,
+            before=before,
+        )
     finally:
         timer.end()
 
-    if not snapshot_pks:
+    if not snapshots.exists():
         log_removal_finished(0, 0)
         raise SystemExit(1)
 
-    if not yes:
-        log_list_finished(snapshots)
-        log_removal_started(snapshots, yes=False)
+    log_list_finished(snapshots)
+    log_removal_started(snapshots, yes=yes)
 
-    from archivebox.search.query import flush_search_index
+    to_remove = snapshots.count()
 
-    started_at = time.monotonic()
-    deadline = started_at + timeout if timeout is not None else None
+    from archivebox.search import flush_search_index
+    from archivebox.core.models import Snapshot
 
-    # Search-index flush touches a separate backend (FTS / sonic), not the
-    # main index.sqlite3 writer lock, so it's safe to do once up front.
     flush_search_index(snapshots=snapshots)
-
-    # Delete one snapshot at a time. Each ``.delete()`` is its own short
-    # Django-atomic block, so the writer lock is released between rows and
-    # an in-flight daemon transaction can interleave instead of deadlocking.
-    # Snapshot.delete() removes and verifies filesystem outputs first, outside
-    # the short database delete transaction. Failed cleanup retains the row.
-    #
-    deleted_snapshot_pks = []
-    timed_out = False
-    timeout_error = ""
-    for index, pk in enumerate(snapshot_pks):
-        if deadline is not None and time.monotonic() >= deadline:
-            timed_out = True
-            timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
-            break
-        deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
-        if deleted_count:
-            deleted_snapshot_pks.append(pk)
-
+    snapshots.delete()
     all_snapshots = Snapshot.objects.all()
-    remaining_count = all_snapshots.count()
-    deleted_snapshot_id_set = set(deleted_snapshot_pks)
-    remaining_snapshot_pks = [snapshot_id for snapshot_id in snapshot_pks if snapshot_id not in deleted_snapshot_id_set]
-    log_removal_finished(remaining_count, len(deleted_snapshot_pks))
+    log_removal_finished(all_snapshots.count(), to_remove)
 
-    return {
-        "removed_count": len(deleted_snapshot_pks),
-        "removed_snapshot_ids": [str(snapshot_id) for snapshot_id in deleted_snapshot_pks],
-        "not_removed_count": len(remaining_snapshot_pks),
-        "not_removed_snapshot_ids": [str(snapshot_id) for snapshot_id in remaining_snapshot_pks],
-        "success": not timed_out,
-        "error": timeout_error,
-        "timeout": timeout,
-    }
+    return all_snapshots
 
 
 @click.command()
 @click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--delete", is_flag=True, help="Compatibility no-op; archived content is deleted automatically")
-@click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
-@snapshot_filter_options(default_filter_type="exact")
+@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
+@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
+@click.option(
+    "--filter-type",
+    "-f",
+    type=click.Choice(("exact", "substring", "domain", "regex", "tag")),
+    default="exact",
+    help="Type of pattern matching to use when filtering URLs",
+)
+@click.argument("filter_patterns", nargs=-1)
 @docstring(remove.__doc__)
 def main(**kwargs):
     """Remove the specified URLs from the archive"""
-    result = remove(**kwargs)
-    if not result["success"]:
-        raise SystemExit(124)
+    remove(**kwargs)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_run.py

@@ -40,25 +40,14 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox run"
 
-import asyncio
-import os
-import signal
 import sys
+import asyncio
 from collections import defaultdict
 
 import rich_click as click
 from rich import print as rprint
 
 
-RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
-
-
-def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
-    # A supervised daemon must exit non-zero so supervisord restarts it, while
-    # still unwinding the active crawl's hooks and persisted process state.
-    raise SystemExit(128 + sig.value)
-
-
 def process_stdin_records() -> int:
     """
     Process JSONL records from stdin.
@@ -69,8 +58,8 @@ def process_stdin_records() -> int:
 
     Outputs JSONL of all processed records (for chaining).
 
-    Handles Crawl and Snapshot work records. ArchiveResult records are accepted
-    as references to their parent Snapshot and plugin.
+    Handles any record type: Crawl, Snapshot, ArchiveResult.
+    Auto-cascades: Crawl → Snapshots → ArchiveResults.
 
     Returns exit code (0 = success, 1 = error).
     """
@@ -87,7 +76,6 @@ def process_stdin_records() -> int:
     )
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot, ArchiveResult
-    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.crawls.models import Crawl
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Binary
@@ -154,15 +142,33 @@ def process_stdin_records() -> int:
                     queued_count += 1
 
             elif record_type == TYPE_ARCHIVERESULT:
-                snapshot_id = str(record.get("snapshot_id") or "")
-                plugin_name = str(record.get("plugin") or "")
-                archiveresult = None
-                if not snapshot_id and record_id:
-                    archiveresult = ArchiveResult.objects.filter(_uuid_ref_query("id", str(record_id))).select_related("snapshot").first()
-                    if archiveresult:
-                        snapshot_id = str(archiveresult.snapshot_id)
-                        plugin_name = plugin_name or archiveresult.plugin
-                snapshot = Snapshot.objects.filter(id=snapshot_id).first() if snapshot_id else None
+                if record_id:
+                    # Existing archiveresult - re-queue
+                    try:
+                        archiveresult = ArchiveResult.objects.get(id=record_id)
+                    except ArchiveResult.DoesNotExist:
+                        archiveresult = None
+                else:
+                    archiveresult = None
+
+                snapshot_id = record.get("snapshot_id")
+                plugin_name = record.get("plugin")
+                snapshot = None
+                if archiveresult:
+                    if archiveresult.status in [
+                        ArchiveResult.StatusChoices.FAILED,
+                        ArchiveResult.StatusChoices.SKIPPED,
+                        ArchiveResult.StatusChoices.NORESULTS,
+                        ArchiveResult.StatusChoices.BACKOFF,
+                    ]:
+                        archiveresult.reset_for_retry()
+                    snapshot = archiveresult.snapshot
+                    plugin_name = plugin_name or archiveresult.plugin
+                elif snapshot_id:
+                    try:
+                        snapshot = Snapshot.objects.get(id=snapshot_id)
+                    except Snapshot.DoesNotExist:
+                        snapshot = None
 
                 if snapshot:
                     snapshot.queue_for_extraction()
@@ -170,7 +176,7 @@ def process_stdin_records() -> int:
                     snapshot_ids_by_crawl[crawl_id].add(str(snapshot.id))
                     if plugin_name:
                         plugin_names_by_crawl[crawl_id].add(str(plugin_name))
-                    output_records.append(archiveresult.to_json() if archiveresult else record)
+                    output_records.append(record if not archiveresult else archiveresult.to_json())
                     queued_count += 1
 
             elif record_type in {TYPE_BINARYREQUEST, TYPE_BINARY}:
@@ -232,11 +238,7 @@ def process_stdin_records() -> int:
     return 0
 
 
-def run_runner(
-    daemon: bool = False,
-    crawl_id: str | None = None,
-    maintenance_only: bool = False,
-) -> int:
+def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int:
     """
     Run the background runner loop.
 
@@ -248,81 +250,37 @@ def run_runner(
     from archivebox.config import CONSTANTS
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Machine, Process
-    from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
-    from archivebox.core.recovery_util import recover_orchestrator_state
-    from archivebox.services.runner import run_pending_crawls
+    from archivebox.services.supervision_service import enter_single_runner_gate
+    from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls
 
     Machine.current()
     current = Process.current()
-    root_command = current.root
-    if daemon and root_command.process_type in (
-        Process.TypeChoices.SERVER,
-        Process.TypeChoices.ADD,
-        Process.TypeChoices.UPDATE,
-    ):
-        # Server-owned daemon runners are persistent supervisor workers, but
-        # foreground add/update commands are allowed to borrow runner/sonic
-        # leadership without taking down Daphne. Waiting here keeps the worker
-        # on the normal runner path while preventing a server restart loop from
-        # immediately stealing the single-runner gate back from the newer CLI.
-        standby_until_foreground_runner_needed(root_command, data_dir=CONSTANTS.DATA_DIR)
     if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
         current.mark_exited()
         return 0
 
-    recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
+    recover_orchestrator_state(include_chrome=True)
     if crawl_id:
         from django.utils import timezone
         from archivebox.crawls.models import Crawl
 
-        crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
-        now = timezone.now()
-        # Winning the single-runner gate terminates and waits for every older
-        # runner. Requeue the explicitly requested crawl so an abandoned
-        # future ownership lease cannot hide it from the unified scheduler.
-        if crawl is not None:
-            crawl.update_and_requeue(retry_at=now, refresh=False)
-    # Interactivity follows terminal ownership, not the spelling of the command.
-    # A supervised add worker has pipes and receives terminal intent/answers from
-    # its parent; direct run owns the terminal itself. Daemons and redirected
-    # commands must never wait for a person who cannot answer their prompt.
-    # Trust the explicit terminal-owner flag, not Process.root: when add borrows
-    # a server-owned supervisord, its worker's OS ancestry still leads to server.
-    # Gating on root=ADD silently made the first Ctrl+C abort after takeover.
-    interactive_interrupts = not daemon and (
-        (sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())) or os.environ.get("ARCHIVEBOX_INTERACTIVE_INTERRUPTS") == "1"
-    )
-    if daemon:
-        os.environ[RUNNER_DAEMON_ENV] = "1"
-
-    shutdown_state = None
+        crawl = Crawl.objects.filter(id=crawl_id, status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED]).first()
+        if crawl is not None and crawl.retry_at is None:
+            crawl.safe_update({"retry_at": timezone.now()}, refresh=False)
+    # Only a foreground `archivebox add` gets the interactive "abort current
+    # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
+    # orchestrators should shut down immediately and cleanly on the first signal.
+    interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
     try:
-        with (
-            foreground_shutdown_signals(
-                on_signal=_exit_daemon_runner_on_signal if daemon else None,
-                raise_on_first_signal=not daemon,
-            ) as shutdown_state,
-            foreground_parent_watchdog(enabled=not daemon),
-        ):
+        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
             run_pending_crawls(
                 daemon=daemon,
                 crawl_id=crawl_id,
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
             )
-        if daemon and shutdown_state.signal_name:
-            return 128 + signal.Signals[shutdown_state.signal_name].value
         return 0
-    except KeyboardInterrupt:
-        if daemon and shutdown_state and shutdown_state.signal_name:
-            return 128 + signal.Signals[shutdown_state.signal_name].value
-        return 130
-    except asyncio.CancelledError as e:
-        if daemon and shutdown_state and shutdown_state.signal_name:
-            return 128 + signal.Signals[shutdown_state.signal_name].value
-        if daemon:
-            rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
-            return 1
+    except (KeyboardInterrupt, asyncio.CancelledError):
         return 0
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
@@ -338,16 +296,8 @@ def run_runner(
 @click.option("--crawl-id", help="Run the crawl runner for a specific crawl only")
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
-@click.option("--maintenance-only", is_flag=True, help="Only process sealed Snapshot maintenance and search-index backfills")
-@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
-def main(
-    daemon: bool,
-    crawl_id: str,
-    snapshot_id: str,
-    binary_id: str,
-    maintenance_only: bool,
-    no_stdin: bool,
-):
+@click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
+def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool):
     """
     Process queued work.
 
@@ -362,21 +312,14 @@ def main(
 
     if daemon and not snapshot_id and not binary_id and not crawl_id:
         try:
-            os.environ[RUNNER_DAEMON_ENV] = "1"
-            with (
-                foreground_shutdown_signals(
-                    on_signal=_exit_daemon_runner_on_signal,
-                    raise_on_first_signal=False,
-                ),
-                foreground_parent_watchdog(enabled=False),
-            ):
+            with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=False):
                 sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
         except KeyboardInterrupt:
-            sys.exit(130)
+            sys.exit(0)
 
     with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
         if snapshot_id:
-            sys.exit(run_snapshot_worker(snapshot_id, daemon=daemon))
+            sys.exit(run_snapshot_worker(snapshot_id))
 
         if binary_id:
             try:
@@ -394,49 +337,30 @@ def main(
                 sys.exit(1)
 
         if crawl_id:
-            sys.exit(
-                run_runner(
-                    daemon=daemon,
-                    crawl_id=crawl_id,
-                    maintenance_only=maintenance_only,
-                ),
-            )
+            sys.exit(run_runner(daemon=False, crawl_id=crawl_id, maintenance_only=maintenance_only))
 
         if maintenance_only:
             sys.exit(run_runner(daemon=daemon, maintenance_only=True))
 
-        if not no_stdin and not sys.stdin.isatty():
+        if not sys.stdin.isatty():
             sys.exit(process_stdin_records())
         else:
             sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
 
 
-def run_snapshot_worker(snapshot_id: str, *, daemon: bool = False) -> int:
-    from archivebox.config import CONSTANTS
-    from archivebox.core.takeover_util import enter_single_runner_gate
+def run_snapshot_worker(snapshot_id: str) -> int:
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-    from archivebox.machine.models import Process
     from archivebox.core.models import Snapshot
     from archivebox.services.runner import run_due_snapshot
     from django.utils import timezone
 
-    current = Process.current()
-    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
-        current.mark_exited()
-        return 0
-
     snapshot = None
-    interactive_interrupts = not daemon and sys.stdin.isatty() and (sys.stdout.isatty() or sys.stderr.isatty())
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
-            for _ in range(10):
-                snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-                if snapshot.retry_at is None:
-                    snapshot.update_and_requeue(retry_at=timezone.now())
-                elif snapshot.retry_at > timezone.now():
-                    break
-                if not run_due_snapshot(snapshot, lock_seconds=60, interactive_interrupts=interactive_interrupts):
-                    break
+            snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+            if snapshot.retry_at is None:
+                snapshot.update_and_requeue(retry_at=timezone.now())
+            run_due_snapshot(snapshot, lock_seconds=60)
         return 0
     except KeyboardInterrupt:
         try:
@@ -448,17 +372,13 @@ def run_snapshot_worker(snapshot_id: str, *, daemon: bool = False) -> int:
                 snapshot.update_and_requeue(retry_at=timezone.now())
         except Exception:
             pass
-        return 130
+        return 0
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
         import traceback
 
         traceback.print_exc()
         return 1
-    finally:
-        current.refresh_from_db()
-        if current.status != Process.StatusChoices.EXITED:
-            current.mark_exited()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_schedule.py

@@ -6,6 +6,7 @@ import rich_click as click
 from rich import print
 
 from archivebox.misc.util import enforce_types, docstring
+from archivebox.config.common import get_config
 
 
 @enforce_types
@@ -19,8 +20,9 @@ def schedule(
     every: str | None = None,
     tag: str = "",
     depth: int | str = 0,
+    overwrite: bool = False,
+    update: bool | None = None,
     import_path: str | None = None,
-    config: dict[str, object] | None = None,
 ):
     """Manage database-backed scheduled crawls processed by the crawl runner."""
 
@@ -28,16 +30,17 @@ def schedule(
 
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl, CrawlSchedule
-    from archivebox.crawls.schedule_util import validate_schedule
+    from archivebox.crawls.schedule_utils import validate_schedule
     from archivebox.services.runner import run_pending_crawls
 
-    config_overrides = dict(config or {})
+    if update is None:
+        update = not get_config().ONLY_NEW
+
     depth = int(depth)
     result: dict[str, object] = {
         "created_schedule_ids": [],
         "disabled_count": 0,
         "run_all_enqueued": 0,
-        "run_all_maintained": 0,
         "active_schedule_ids": [],
     }
 
@@ -58,12 +61,12 @@ def schedule(
 
         created_by_id = get_or_create_system_user_pk()
         is_update_schedule = not import_path
-        template_urls = import_path or ""
+        template_urls = import_path or "archivebox://update"
         template_label = (f"Scheduled import: {template_urls}" if import_path else "Scheduled ArchiveBox update")[:64]
         template_notes = (
             f"Created by archivebox schedule for {template_urls}"
             if import_path
-            else "Created by archivebox schedule to run recurring ArchiveBox maintenance."
+            else "Created by archivebox schedule to queue recurring archivebox://update maintenance crawls."
         )
 
         template = Crawl.objects.create(
@@ -76,17 +79,15 @@ def schedule(
             status=Crawl.StatusChoices.SEALED,
             retry_at=None,
             config={
+                "ONLY_NEW": not update,
+                "OVERWRITE": overwrite,
                 "DEPTH": 0 if is_update_schedule else depth,
-                # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) win over the
-                # template defaults. Anything left unset falls through to the
-                # standard config stack at crawl-resolution time.
-                **config_overrides,
+                "SCHEDULE_KIND": "update" if is_update_schedule else "crawl",
             },
         )
         crawl_schedule = CrawlSchedule.objects.create(
             template=template,
             schedule=schedule_str,
-            config={**template.config, "SCHEDULE_KIND": "update" if is_update_schedule else "crawl"},
             is_enabled=True,
             label=template_label,
             notes=template_notes,
@@ -120,19 +121,12 @@ def schedule(
 
     if run_all:
         enqueued = 0
-        maintained = 0
         now = timezone.now()
         for scheduled_crawl in schedules:
-            if scheduled_crawl.dispatch(queued_at=now) is None:
-                maintained += 1
-            else:
-                enqueued += 1
+            scheduled_crawl.enqueue(queued_at=now)
+            enqueued += 1
         result["run_all_enqueued"] = enqueued
-        result["run_all_maintained"] = maintained
         print(f"[green]\\[*] Enqueued {enqueued} scheduled crawl(s) immediately.[/green]")
-        if maintained:
-            run_pending_crawls(maintenance_only=True)
-            print(f"[green]\\[*] Ran {maintained} scheduled maintenance update(s).[/green]")
         if enqueued:
             print(
                 "[yellow]\\[*] Start `archivebox server`, `archivebox run --daemon`, or `archivebox schedule --foreground` to process the queued crawls.[/yellow]",
@@ -169,24 +163,16 @@ def schedule(
     default="0",
     help="Recursively archive linked pages up to N hops away",
 )
-@click.option(
-    "--only-new/--no-only-new",
-    "only_new",
-    default=None,
-    help="Skip URLs that already have a snapshot (default: inherit from ONLY_NEW config). "
-    "Pass --no-only-new to force re-archive on each scheduled run.",
-)
+@click.option("--overwrite", is_flag=True, help="Overwrite existing data if URLs have been archived previously")
+@click.option("--update", is_flag=True, help="Retry previously failed/skipped URLs when scheduled crawls run")
 @click.option("--clear", is_flag=True, help="Disable all currently enabled schedules")
 @click.option("--show", is_flag=True, help="Print all currently enabled schedules")
 @click.option("--foreground", "-f", is_flag=True, help="Run the global crawl runner in the foreground (no crontab required)")
-@click.option("--run-all", is_flag=True, help="Dispatch all enabled schedules immediately")
+@click.option("--run-all", is_flag=True, help="Enqueue all enabled schedules immediately and process them once")
 @click.argument("import_path", required=False)
 @docstring(schedule.__doc__)
 def main(**kwargs):
     """Manage database-backed scheduled crawls processed by the crawl runner."""
-    only_new = kwargs.pop("only_new", None)
-    if only_new is not None:
-        kwargs["config"] = {"ONLY_NEW": bool(only_new)}
     schedule(**kwargs)
 
 

archivebox/cli/archivebox_search.py

@@ -3,7 +3,255 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox search"
 
-from archivebox.cli.archivebox_list import main
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+from collections.abc import Callable
+
+import rich_click as click
+
+from django.db.models import Q, QuerySet
+
+from archivebox.config import DATA_DIR
+from archivebox.config.common import get_config
+from archivebox.misc.logging import stderr
+from archivebox.misc.util import enforce_types, docstring
+
+if TYPE_CHECKING:
+    from archivebox.core.models import Snapshot
+
+# Filter types for URL matching
+LINK_FILTERS: dict[str, Callable[[str], Q]] = {
+    "exact": lambda pattern: Q(url=pattern),
+    "substring": lambda pattern: Q(url__icontains=pattern),
+    "regex": lambda pattern: Q(url__iregex=pattern),
+    "domain": lambda pattern: (
+        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
+    ),
+    "tag": lambda pattern: Q(tags__name=pattern),
+    "timestamp": lambda pattern: Q(timestamp=pattern),
+}
+
+STATUS_CHOICES = ["indexed", "archived", "unarchived"]
+
+
+def _apply_pattern_filters(
+    snapshots: QuerySet["Snapshot", "Snapshot"],
+    filter_patterns: list[str],
+    filter_type: str,
+) -> QuerySet["Snapshot", "Snapshot"]:
+    filter_builder = LINK_FILTERS.get(filter_type)
+    if filter_builder is None:
+        stderr()
+        stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red")
+        raise SystemExit(2)
+
+    query = Q()
+    for pattern in filter_patterns:
+        query |= filter_builder(pattern)
+    return snapshots.filter(query)
+
+
+def _snapshots_to_json(
+    snapshots: QuerySet["Snapshot", "Snapshot"],
+    *,
+    with_headers: bool,
+) -> str:
+    from datetime import datetime, timezone as tz
+
+    from archivebox.config import VERSION
+    from archivebox.misc.util import to_json
+
+    config = get_config()
+    main_index_header = (
+        {
+            "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
+            "schema": "archivebox.index.json",
+            "copyright_info": config.FOOTER_INFO,
+            "meta": {
+                "project": "ArchiveBox",
+                "version": VERSION,
+                "git_sha": VERSION,
+                "website": "https://ArchiveBox.io",
+                "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
+                "source": "https://github.com/ArchiveBox/ArchiveBox",
+                "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
+                "dependencies": {},
+            },
+        }
+        if with_headers
+        else {}
+    )
+
+    snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)]
+    output: dict[str, object] | list[dict[str, object]]
+    if with_headers:
+        output = {
+            **main_index_header,
+            "num_links": len(snapshot_dicts),
+            "updated": datetime.now(tz.utc),
+            "last_run_cmd": sys.argv,
+            "links": snapshot_dicts,
+        }
+    else:
+        output = snapshot_dicts
+
+    return to_json(output, indent=4, sort_keys=True)
+
+
+def _snapshots_to_csv(
+    snapshots: QuerySet["Snapshot", "Snapshot"],
+    *,
+    cols: list[str],
+    with_headers: bool,
+) -> str:
+    header = ",".join(cols) if with_headers else ""
+    rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)]
+    return "\n".join((header, *rows))
+
+
+def _snapshots_to_html(
+    snapshots: QuerySet["Snapshot", "Snapshot"],
+    *,
+    with_headers: bool,
+) -> str:
+    from datetime import datetime, timezone as tz
+
+    from django.template.loader import render_to_string
+
+    from archivebox.config import VERSION
+    from archivebox.config.version import get_COMMIT_HASH
+
+    config = get_config()
+    template = "static_index.html" if with_headers else "minimal_index.html"
+    snapshot_list = list(snapshots.iterator(chunk_size=500))
+
+    return render_to_string(
+        template,
+        {
+            "version": VERSION,
+            "git_sha": get_COMMIT_HASH() or VERSION,
+            "num_links": str(len(snapshot_list)),
+            "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
+            "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
+            "links": snapshot_list,
+            "FOOTER_INFO": config.FOOTER_INFO,
+        },
+    )
+
+
+def get_snapshots(
+    snapshots: QuerySet["Snapshot", "Snapshot"] | None = None,
+    filter_patterns: list[str] | None = None,
+    filter_type: str = "substring",
+    after: float | None = None,
+    before: float | None = None,
+    out_dir: Path = DATA_DIR,
+) -> QuerySet["Snapshot", "Snapshot"]:
+    """Filter and return Snapshots matching the given criteria."""
+    from archivebox.core.models import Snapshot
+
+    if snapshots is not None:
+        result = snapshots
+    else:
+        result = Snapshot.objects.all()
+
+    if after is not None:
+        result = result.filter(timestamp__gte=after)
+    if before is not None:
+        result = result.filter(timestamp__lt=before)
+    if filter_patterns:
+        result = _apply_pattern_filters(result, filter_patterns, filter_type)
+
+    # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir
+    result = result.select_related("crawl", "crawl__created_by")
+
+    if not result.exists():
+        stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow")
+
+    return result
+
+
+@enforce_types
+def search(
+    filter_patterns: list[str] | None = None,
+    filter_type: str = "substring",
+    status: str = "indexed",
+    before: float | None = None,
+    after: float | None = None,
+    sort: str | None = None,
+    json: bool = False,
+    html: bool = False,
+    csv: str | None = None,
+    with_headers: bool = False,
+):
+    """List, filter, and export information about archive entries"""
+
+    if with_headers and not (json or html or csv):
+        stderr("[X] --with-headers requires --json, --html or --csv\n", color="red")
+        raise SystemExit(2)
+
+    # Query DB directly - no filesystem scanning
+    snapshots = get_snapshots(
+        filter_patterns=list(filter_patterns) if filter_patterns else None,
+        filter_type=filter_type,
+        before=before,
+        after=after,
+    )
+
+    # Apply status filter
+    if status == "archived":
+        snapshots = snapshots.filter(downloaded_at__isnull=False)
+    elif status == "unarchived":
+        snapshots = snapshots.filter(downloaded_at__isnull=True)
+    # 'indexed' = all snapshots (no filter)
+
+    if sort:
+        snapshots = snapshots.order_by(sort)
+
+    # Export to requested format
+    if json:
+        output = _snapshots_to_json(snapshots, with_headers=with_headers)
+    elif html:
+        output = _snapshots_to_html(snapshots, with_headers=with_headers)
+    elif csv:
+        output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers)
+    else:
+        from archivebox.misc.logging_util import printable_folders
+
+        # Convert to dict for printable_folders
+        folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots}
+        output = printable_folders(folders, with_headers)
+
+    # Structured exports must be written directly to stdout.
+    # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output.
+    sys.stdout.write(output)
+    if not output.endswith("\n"):
+        sys.stdout.write("\n")
+    return output
+
+
+@click.command()
+@click.option(
+    "--filter-type",
+    "-f",
+    type=click.Choice(["search", *LINK_FILTERS.keys()]),
+    default="substring",
+    help="Pattern matching type for filtering URLs",
+)
+@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status")
+@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp")
+@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp")
+@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
+@click.option("--json", "-J", is_flag=True, help="Print output in JSON format")
+@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)")
+@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title")
+@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output")
+@click.help_option("--help", "-h")
+@click.argument("filter_patterns", nargs=-1)
+@docstring(search.__doc__)
+def main(**kwargs):
+    return search(**kwargs)
 
 
 if __name__ == "__main__":